diff --git "a/metrics.jsonl" "b/metrics.jsonl"
--- "a/metrics.jsonl"
+++ "b/metrics.jsonl"
@@ -1,3492 +1,3 @@
-{"step": 1, "tokens": 106669, "loss_tokens": 106669, "loss": 10.51152631036196, "lr": 1.6000499999999998e-06, "optimizer_lrs": [0.00010667, 1.6000499999999998e-06], "gradient_norm": 4.611739635467529, "seconds": 135.83415617999998, "nonpadding_tokens_per_second": 785.2884944391166, "ranks": [{"compute_seconds": 134.522632636, "nonpadding_tokens": 9227, "peak_allocated_bytes": 8000962560, "peak_reserved_bytes": 9017753600}, {"compute_seconds": 134.522949873, "nonpadding_tokens": 2394, "peak_allocated_bytes": 4455890944, "peak_reserved_bytes": 4869586944}, {"compute_seconds": 134.522880742, "nonpadding_tokens": 17786, "peak_allocated_bytes": 14373146112, "peak_reserved_bytes": 17400070144}, {"compute_seconds": 134.52499360000002, "nonpadding_tokens": 13566, "peak_allocated_bytes": 13833286144, "peak_reserved_bytes": 15562964992}, {"compute_seconds": 134.528257375, "nonpadding_tokens": 12266, "peak_allocated_bytes": 13833286144, "peak_reserved_bytes": 15562964992}, {"compute_seconds": 134.52280045700002, "nonpadding_tokens": 9683, "peak_allocated_bytes": 10641830400, "peak_reserved_bytes": 12035555328}, {"compute_seconds": 134.522533591, "nonpadding_tokens": 21737, "peak_allocated_bytes": 14376860160, "peak_reserved_bytes": 17710448640}, {"compute_seconds": 134.52182172, "nonpadding_tokens": 20010, "peak_allocated_bytes": 13833286144, "peak_reserved_bytes": 15562964992}], "data_fetch_seconds": 0.0789631630000116, "prefetch": {"data_wait_seconds": 1.6629999990414035e-05, "consumed_batches": 1, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2, "tokens": 198806, "loss_tokens": 92137, "loss": 10.48297304557344, "lr": 2.9821049999999997e-06, "optimizer_lrs": [0.00019880699999999998, 2.9821049999999997e-06], "gradient_norm": 4.935856819152832, "seconds": 48.811896563000005, "nonpadding_tokens_per_second": 1887.593117409024, "ranks": [{"compute_seconds": 48.78181791000003, "nonpadding_tokens": 5735, "peak_allocated_bytes": 8585426944, "peak_reserved_bytes": 10068426752}, {"compute_seconds": 48.76506755699998, "nonpadding_tokens": 4697, "peak_allocated_bytes": 6117104128, "peak_reserved_bytes": 6757023744}, {"compute_seconds": 48.766729492999985, "nonpadding_tokens": 10706, "peak_allocated_bytes": 14976782848, "peak_reserved_bytes": 17402167296}, {"compute_seconds": 48.76641644700001, "nonpadding_tokens": 14130, "peak_allocated_bytes": 14470052352, "peak_reserved_bytes": 17710448640}, {"compute_seconds": 48.794246723000015, "nonpadding_tokens": 8539, "peak_allocated_bytes": 13833286144, "peak_reserved_bytes": 15562964992}, {"compute_seconds": 48.79414803999998, "nonpadding_tokens": 17230, "peak_allocated_bytes": 14987047424, "peak_reserved_bytes": 18486394880}, {"compute_seconds": 48.765559713000016, "nonpadding_tokens": 10581, "peak_allocated_bytes": 14976948736, "peak_reserved_bytes": 17710448640}, {"compute_seconds": 48.770267030999975, "nonpadding_tokens": 20519, "peak_allocated_bytes": 14470052352, "peak_reserved_bytes": 17710448640}], "data_fetch_seconds": 0.00046847300001218173, "prefetch": {"data_wait_seconds": 3.175799999155515e-05, "consumed_batches": 2, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3, "tokens": 303408, "loss_tokens": 104602, "loss": 10.502557312479684, "lr": 4.551134999999999e-06, "optimizer_lrs": [0.000303409, 4.551134999999999e-06], "gradient_norm": 3.892406940460205, "seconds": 64.868105243, "nonpadding_tokens_per_second": 1612.5336112123875, "ranks": [{"compute_seconds": 63.646571832999996, "nonpadding_tokens": 9539, "peak_allocated_bytes": 11967026176, "peak_reserved_bytes": 12952010752}, {"compute_seconds": 63.58657947300003, "nonpadding_tokens": 14541, "peak_allocated_bytes": 8550326784, "peak_reserved_bytes": 10313793536}, {"compute_seconds": 63.60453268100002, "nonpadding_tokens": 21991, "peak_allocated_bytes": 14965696000, "peak_reserved_bytes": 17739808768}, {"compute_seconds": 63.586608883999986, "nonpadding_tokens": 12248, "peak_allocated_bytes": 14972539392, "peak_reserved_bytes": 17559453696}, {"compute_seconds": 63.59821804799998, "nonpadding_tokens": 5739, "peak_allocated_bytes": 6933766656, "peak_reserved_bytes": 7602176000}, {"compute_seconds": 63.58613828099999, "nonpadding_tokens": 20207, "peak_allocated_bytes": 14965696000, "peak_reserved_bytes": 17739808768}, {"compute_seconds": 63.586830292, "nonpadding_tokens": 13309, "peak_allocated_bytes": 14428038656, "peak_reserved_bytes": 15592325120}, {"compute_seconds": 63.603220040999986, "nonpadding_tokens": 7028, "peak_allocated_bytes": 7150583296, "peak_reserved_bytes": 7772045312}], "data_fetch_seconds": 0.031902387000002363, "prefetch": {"data_wait_seconds": 1.8764000003557157e-05, "consumed_batches": 1, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 4, "tokens": 387014, "loss_tokens": 83606, "loss": 10.484481526445471, "lr": 5.805224999999999e-06, "optimizer_lrs": [0.000387015, 5.805224999999999e-06], "gradient_norm": 4.556110858917236, "seconds": 29.22900930000003, "nonpadding_tokens_per_second": 2860.3774812169195, "ranks": [{"compute_seconds": 29.226520249000032, "nonpadding_tokens": 7791, "peak_allocated_bytes": 11967026176, "peak_reserved_bytes": 12952010752}, {"compute_seconds": 29.225964660000045, "nonpadding_tokens": 10596, "peak_allocated_bytes": 13927245824, "peak_reserved_bytes": 16961765376}, {"compute_seconds": 29.221560974, "nonpadding_tokens": 5439, "peak_allocated_bytes": 14965696000, "peak_reserved_bytes": 17739808768}, {"compute_seconds": 29.222303141999987, "nonpadding_tokens": 17200, "peak_allocated_bytes": 14975798784, "peak_reserved_bytes": 19706937344}, {"compute_seconds": 29.221442461000038, "nonpadding_tokens": 2952, "peak_allocated_bytes": 6933766656, "peak_reserved_bytes": 7602176000}, {"compute_seconds": 29.224220237999987, "nonpadding_tokens": 17389, "peak_allocated_bytes": 14971584000, "peak_reserved_bytes": 17741905920}, {"compute_seconds": 29.222185335999995, "nonpadding_tokens": 12385, "peak_allocated_bytes": 14463166976, "peak_reserved_bytes": 17739808768}, {"compute_seconds": 29.221855285999993, "nonpadding_tokens": 9854, "peak_allocated_bytes": 8633186304, "peak_reserved_bytes": 12419334144}], "data_fetch_seconds": 0.00037605400001439193, "prefetch": {"data_wait_seconds": 3.4210000023904286e-05, "consumed_batches": 2, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 5, "tokens": 457369, "loss_tokens": 70355, "loss": 10.491991507355554, "lr": 6.860549999999999e-06, "optimizer_lrs": [0.00045736999999999997, 6.860549999999999e-06], "gradient_norm": 4.3774943351745605, "seconds": 84.75310046200002, "nonpadding_tokens_per_second": 830.117123933943, "ranks": [{"compute_seconds": 83.41909913699999, "nonpadding_tokens": 13669, "peak_allocated_bytes": 14455694848, "peak_reserved_bytes": 15986589696}, {"compute_seconds": 83.41941136799994, "nonpadding_tokens": 10815, "peak_allocated_bytes": 14974306816, "peak_reserved_bytes": 17500733440}, {"compute_seconds": 83.41939483700003, "nonpadding_tokens": 11776, "peak_allocated_bytes": 14969768448, "peak_reserved_bytes": 17978884096}, {"compute_seconds": 83.41952429700007, "nonpadding_tokens": 10621, "peak_allocated_bytes": 14924346880, "peak_reserved_bytes": 16487809024}, {"compute_seconds": 83.41940043500006, "nonpadding_tokens": 5145, "peak_allocated_bytes": 5298070528, "peak_reserved_bytes": 5733613568}, {"compute_seconds": 83.41980655599991, "nonpadding_tokens": 6679, "peak_allocated_bytes": 7800869376, "peak_reserved_bytes": 8453619712}, {"compute_seconds": 83.41944219999993, "nonpadding_tokens": 3004, "peak_allocated_bytes": 4695237632, "peak_reserved_bytes": 5104467968}, {"compute_seconds": 83.41848887699996, "nonpadding_tokens": 8646, "peak_allocated_bytes": 12403363328, "peak_reserved_bytes": 13660848128}], "data_fetch_seconds": 0.0802614880000192, "prefetch": {"data_wait_seconds": 1.93089999811491e-05, "consumed_batches": 1, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 6, "tokens": 559176, "loss_tokens": 101807, "loss": 10.465074847505575, "lr": 8.387655e-06, "optimizer_lrs": [0.000559177, 8.387655e-06], "gradient_norm": 3.7638204097747803, "seconds": 0.409752989000026, "nonpadding_tokens_per_second": 248459.44442883256, "ranks": [{"compute_seconds": 0.4038595559999294, "nonpadding_tokens": 22919, "peak_allocated_bytes": 14489095680, "peak_reserved_bytes": 15986589696}, {"compute_seconds": 0.40588178699999844, "nonpadding_tokens": 16096, "peak_allocated_bytes": 14974306816, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.40512022000007164, "nonpadding_tokens": 6035, "peak_allocated_bytes": 14969768448, "peak_reserved_bytes": 17978884096}, {"compute_seconds": 0.40215462999992724, "nonpadding_tokens": 8619, "peak_allocated_bytes": 14924346880, "peak_reserved_bytes": 16487809024}, {"compute_seconds": 0.40436035100003664, "nonpadding_tokens": 10188, "peak_allocated_bytes": 14462546432, "peak_reserved_bytes": 17005805568}, {"compute_seconds": 0.40188534700007494, "nonpadding_tokens": 6963, "peak_allocated_bytes": 7800869376, "peak_reserved_bytes": 8453619712}, {"compute_seconds": 0.4030179489999455, "nonpadding_tokens": 16051, "peak_allocated_bytes": 14463819264, "peak_reserved_bytes": 17433624576}, {"compute_seconds": 0.4048198709999724, "nonpadding_tokens": 14936, "peak_allocated_bytes": 14463429120, "peak_reserved_bytes": 19033751552}], "data_fetch_seconds": 0.0004916489999686746, "prefetch": {"data_wait_seconds": 3.533400001742848e-05, "consumed_batches": 2, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 7, "tokens": 659656, "loss_tokens": 100480, "loss": 10.460373955015923, "lr": 9.894855e-06, "optimizer_lrs": [0.000659657, 9.894855e-06], "gradient_norm": 3.598740577697754, "seconds": 0.33155253400002493, "nonpadding_tokens_per_second": 303059.06212736847, "ranks": [{"compute_seconds": 0.32726492300002974, "nonpadding_tokens": 14432, "peak_allocated_bytes": 14489095680, "peak_reserved_bytes": 15986589696}, {"compute_seconds": 0.3291084720000299, "nonpadding_tokens": 11743, "peak_allocated_bytes": 14974306816, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.32862252200004605, "nonpadding_tokens": 10429, "peak_allocated_bytes": 14969768448, "peak_reserved_bytes": 17978884096}, {"compute_seconds": 0.3255725510000502, "nonpadding_tokens": 12797, "peak_allocated_bytes": 14924346880, "peak_reserved_bytes": 16487809024}, {"compute_seconds": 0.32653517699998247, "nonpadding_tokens": 10167, "peak_allocated_bytes": 14462546432, "peak_reserved_bytes": 17005805568}, {"compute_seconds": 0.3261230450000312, "nonpadding_tokens": 10680, "peak_allocated_bytes": 14465133056, "peak_reserved_bytes": 16968056832}, {"compute_seconds": 0.3291280700000243, "nonpadding_tokens": 22692, "peak_allocated_bytes": 14976188928, "peak_reserved_bytes": 19581108224}, {"compute_seconds": 0.3292274589999806, "nonpadding_tokens": 7540, "peak_allocated_bytes": 14463429120, "peak_reserved_bytes": 19033751552}], "data_fetch_seconds": 0.0003534920000447528, "prefetch": {"data_wait_seconds": 4.458099999737897e-05, "consumed_batches": 3, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 8, "tokens": 773307, "loss_tokens": 113651, "loss": 10.374868016999411, "lr": 1.159962e-05, "optimizer_lrs": [0.000773308, 1.159962e-05], "gradient_norm": 4.919204235076904, "seconds": 0.39837875900002473, "nonpadding_tokens_per_second": 285283.7844198238, "ranks": [{"compute_seconds": 0.3941015460000017, "nonpadding_tokens": 16078, "peak_allocated_bytes": 14744635904, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.3955034399999704, "nonpadding_tokens": 19137, "peak_allocated_bytes": 14979168768, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3949729840001055, "nonpadding_tokens": 7425, "peak_allocated_bytes": 14969768448, "peak_reserved_bytes": 17978884096}, {"compute_seconds": 0.39588343100001566, "nonpadding_tokens": 9338, "peak_allocated_bytes": 14924346880, "peak_reserved_bytes": 16487809024}, {"compute_seconds": 0.3921353380000028, "nonpadding_tokens": 5205, "peak_allocated_bytes": 14462546432, "peak_reserved_bytes": 17081303040}, {"compute_seconds": 0.39176509000003534, "nonpadding_tokens": 30355, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19115540480}, {"compute_seconds": 0.39430237900000975, "nonpadding_tokens": 13125, "peak_allocated_bytes": 14976188928, "peak_reserved_bytes": 19581108224}, {"compute_seconds": 0.39503590300000724, "nonpadding_tokens": 12988, "peak_allocated_bytes": 14970810880, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0005235310001125981, "prefetch": {"data_wait_seconds": 5.474200008848129e-05, "consumed_batches": 4, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 9, "tokens": 865883, "loss_tokens": 92576, "loss": 10.358699879018321, "lr": 1.2988259999999998e-05, "optimizer_lrs": [0.0008658839999999999, 1.2988259999999998e-05], "gradient_norm": 3.6846437454223633, "seconds": 0.4194240379999883, "nonpadding_tokens_per_second": 220721.73173823333, "ranks": [{"compute_seconds": 0.39524623099998735, "nonpadding_tokens": 26113, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.4170433369999955, "nonpadding_tokens": 5695, "peak_allocated_bytes": 14979168768, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.41246405999993385, "nonpadding_tokens": 5945, "peak_allocated_bytes": 14969768448, "peak_reserved_bytes": 17978884096}, {"compute_seconds": 0.4168109800000366, "nonpadding_tokens": 6430, "peak_allocated_bytes": 14924346880, "peak_reserved_bytes": 16487809024}, {"compute_seconds": 0.41244082200000776, "nonpadding_tokens": 20812, "peak_allocated_bytes": 14462546432, "peak_reserved_bytes": 17081303040}, {"compute_seconds": 0.4129882660000703, "nonpadding_tokens": 18022, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19115540480}, {"compute_seconds": 0.4158782550000524, "nonpadding_tokens": 3122, "peak_allocated_bytes": 14976188928, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.4164538210000046, "nonpadding_tokens": 6437, "peak_allocated_bytes": 14970810880, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00039185299999644485, "prefetch": {"data_wait_seconds": 6.413300013718981e-05, "consumed_batches": 5, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 10, "tokens": 959099, "loss_tokens": 93216, "loss": 10.281412257552352, "lr": 1.4386499999999999e-05, "optimizer_lrs": [0.0009591, 1.4386499999999999e-05], "gradient_norm": 5.120384693145752, "seconds": 0.356939978000014, "nonpadding_tokens_per_second": 261153.15107683552, "ranks": [{"compute_seconds": 0.3382452910000211, "nonpadding_tokens": 24315, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.34320837600000687, "nonpadding_tokens": 12731, "peak_allocated_bytes": 14979168768, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.34338032100004057, "nonpadding_tokens": 6241, "peak_allocated_bytes": 14969768448, "peak_reserved_bytes": 18018729984}, {"compute_seconds": 0.34288677899996856, "nonpadding_tokens": 12685, "peak_allocated_bytes": 14924346880, "peak_reserved_bytes": 16487809024}, {"compute_seconds": 0.343969560000005, "nonpadding_tokens": 11070, "peak_allocated_bytes": 14462546432, "peak_reserved_bytes": 17081303040}, {"compute_seconds": 0.3433083009999791, "nonpadding_tokens": 6496, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19115540480}, {"compute_seconds": 0.3463571830000092, "nonpadding_tokens": 10076, "peak_allocated_bytes": 14976188928, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3461870280000312, "nonpadding_tokens": 9602, "peak_allocated_bytes": 14970810880, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00041346600005454093, "prefetch": {"data_wait_seconds": 7.325100011712493e-05, "consumed_batches": 6, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 11, "tokens": 1024760, "loss_tokens": 65661, "loss": 10.268721158678668, "lr": 1.5371414999999998e-05, "optimizer_lrs": [0.001024761, 1.5371414999999998e-05], "gradient_norm": 4.011878490447998, "seconds": 0.31347090899998875, "nonpadding_tokens_per_second": 209464.41317143835, "ranks": [{"compute_seconds": 0.23594450900009178, "nonpadding_tokens": 4690, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.2582109960000025, "nonpadding_tokens": 9717, "peak_allocated_bytes": 14979168768, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.25587465900002826, "nonpadding_tokens": 5144, "peak_allocated_bytes": 14969768448, "peak_reserved_bytes": 18018729984}, {"compute_seconds": 0.2592199780000328, "nonpadding_tokens": 6335, "peak_allocated_bytes": 14924346880, "peak_reserved_bytes": 16487809024}, {"compute_seconds": 0.2575375309999117, "nonpadding_tokens": 18526, "peak_allocated_bytes": 14965007872, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.2704268059999322, "nonpadding_tokens": 3231, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.26718860300002234, "nonpadding_tokens": 7232, "peak_allocated_bytes": 14976188928, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.25737766899999315, "nonpadding_tokens": 10786, "peak_allocated_bytes": 14970810880, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.013515740000002552, "prefetch": {"data_wait_seconds": 8.38630000998819e-05, "consumed_batches": 7, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 12, "tokens": 1098012, "loss_tokens": 73252, "loss": 10.184174152241576, "lr": 1.6470195e-05, "optimizer_lrs": [0.0010980130000000001, 1.6470195e-05], "gradient_norm": 3.953291416168213, "seconds": 0.2891203020000148, "nonpadding_tokens_per_second": 253361.66119526344, "ranks": [{"compute_seconds": 0.2867720090000603, "nonpadding_tokens": 3864, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.2812394189999168, "nonpadding_tokens": 11359, "peak_allocated_bytes": 14979168768, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.2808785299999954, "nonpadding_tokens": 12257, "peak_allocated_bytes": 14973073920, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.28242777200000546, "nonpadding_tokens": 11915, "peak_allocated_bytes": 14924346880, "peak_reserved_bytes": 20782776320}, {"compute_seconds": 0.279080009999916, "nonpadding_tokens": 3379, "peak_allocated_bytes": 14965007872, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.27853410299996995, "nonpadding_tokens": 5889, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.28147130799993647, "nonpadding_tokens": 13770, "peak_allocated_bytes": 14976188928, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.2868301910000355, "nonpadding_tokens": 10819, "peak_allocated_bytes": 14970810880, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0004107590000330674, "prefetch": {"data_wait_seconds": 9.383800011164567e-05, "consumed_batches": 8, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 13, "tokens": 1234400, "loss_tokens": 136388, "loss": 10.154563634630613, "lr": 1.8516014999999998e-05, "optimizer_lrs": [0.001234401, 1.8516014999999998e-05], "gradient_norm": 4.1166157722473145, "seconds": 0.484164569000086, "nonpadding_tokens_per_second": 281697.60600548156, "ranks": [{"compute_seconds": 0.47820237299993096, "nonpadding_tokens": 16649, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.48072857800002566, "nonpadding_tokens": 18952, "peak_allocated_bytes": 14986164736, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.4773539040000969, "nonpadding_tokens": 19196, "peak_allocated_bytes": 14973073920, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.4818140529999937, "nonpadding_tokens": 14166, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20782776320}, {"compute_seconds": 0.47760264500004723, "nonpadding_tokens": 20465, "peak_allocated_bytes": 14965007872, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.4776712299999417, "nonpadding_tokens": 17984, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.4784832960000358, "nonpadding_tokens": 19002, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.47966635399995994, "nonpadding_tokens": 9974, "peak_allocated_bytes": 14970810880, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003593019999925673, "prefetch": {"data_wait_seconds": 0.00010289000010743621, "consumed_batches": 9, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 14, "tokens": 1327370, "loss_tokens": 92970, "loss": 10.151250403355922, "lr": 1.9910564999999996e-05, "optimizer_lrs": [0.001327371, 1.9910564999999996e-05], "gradient_norm": 3.504115104675293, "seconds": 0.2803734980000172, "nonpadding_tokens_per_second": 331593.39475086296, "ranks": [{"compute_seconds": 0.27754739699992115, "nonpadding_tokens": 8102, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.27811398699998335, "nonpadding_tokens": 13034, "peak_allocated_bytes": 14986164736, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.2773575119999805, "nonpadding_tokens": 26772, "peak_allocated_bytes": 14975105536, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.2777422999999999, "nonpadding_tokens": 10435, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20820525056}, {"compute_seconds": 0.2751156780000201, "nonpadding_tokens": 11308, "peak_allocated_bytes": 14965007872, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.27393782900003316, "nonpadding_tokens": 5714, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.2777032420000296, "nonpadding_tokens": 6947, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.27730985599998803, "nonpadding_tokens": 10658, "peak_allocated_bytes": 14970810880, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003539929999760716, "prefetch": {"data_wait_seconds": 0.00011223500007417897, "consumed_batches": 10, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 15, "tokens": 1411145, "loss_tokens": 83775, "loss": 10.030322291853178, "lr": 2.116719e-05, "optimizer_lrs": [0.001411146, 2.116719e-05], "gradient_norm": 3.4246158599853516, "seconds": 0.3923205410000037, "nonpadding_tokens_per_second": 213537.12397128655, "ranks": [{"compute_seconds": 0.3878661029999648, "nonpadding_tokens": 3364, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.38676520800004255, "nonpadding_tokens": 11644, "peak_allocated_bytes": 14986164736, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.38661852500001714, "nonpadding_tokens": 8613, "peak_allocated_bytes": 14975105536, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.3869319750000386, "nonpadding_tokens": 12277, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20820525056}, {"compute_seconds": 0.38985077100005583, "nonpadding_tokens": 21632, "peak_allocated_bytes": 14966105600, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.38693826899998385, "nonpadding_tokens": 6670, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.3899658359999876, "nonpadding_tokens": 7004, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3894391740000174, "nonpadding_tokens": 12571, "peak_allocated_bytes": 14970810880, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00038495700005114486, "prefetch": {"data_wait_seconds": 0.00012097500007257622, "consumed_batches": 11, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 16, "tokens": 1503444, "loss_tokens": 92299, "loss": 10.078121106404186, "lr": 2.2551674999999998e-05, "optimizer_lrs": [0.001503445, 2.2551674999999998e-05], "gradient_norm": 3.079272747039795, "seconds": 0.32877650700004324, "nonpadding_tokens_per_second": 280734.77889947855, "ranks": [{"compute_seconds": 0.32516487600003074, "nonpadding_tokens": 6473, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.3210665240000026, "nonpadding_tokens": 10928, "peak_allocated_bytes": 14986164736, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.32455008600004476, "nonpadding_tokens": 7661, "peak_allocated_bytes": 14975105536, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.3258543079999754, "nonpadding_tokens": 5397, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3242803890000232, "nonpadding_tokens": 4851, "peak_allocated_bytes": 14966105600, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.32634083099992495, "nonpadding_tokens": 24212, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.3247696649999625, "nonpadding_tokens": 12690, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.32405628399999387, "nonpadding_tokens": 20087, "peak_allocated_bytes": 14970810880, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003673940000226139, "prefetch": {"data_wait_seconds": 0.0001303590000816257, "consumed_batches": 12, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 17, "tokens": 1616293, "loss_tokens": 112849, "loss": 10.003277609903499, "lr": 2.424441e-05, "optimizer_lrs": [0.001616294, 2.424441e-05], "gradient_norm": 3.2472267150878906, "seconds": 0.34128052099993056, "nonpadding_tokens_per_second": 330663.4661402869, "ranks": [{"compute_seconds": 0.33576699100001406, "nonpadding_tokens": 9744, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.3355093200000283, "nonpadding_tokens": 12441, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3388716499999873, "nonpadding_tokens": 14921, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.3386824010000282, "nonpadding_tokens": 2617, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3380679050000026, "nonpadding_tokens": 11699, "peak_allocated_bytes": 14966105600, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.33546417600007317, "nonpadding_tokens": 19171, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.3365618190000532, "nonpadding_tokens": 21674, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.33830007899996417, "nonpadding_tokens": 20582, "peak_allocated_bytes": 14972038656, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00035008299994387926, "prefetch": {"data_wait_seconds": 0.00013970100008009467, "consumed_batches": 13, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 18, "tokens": 1724982, "loss_tokens": 108689, "loss": 9.941150898434984, "lr": 2.5874744999999998e-05, "optimizer_lrs": [0.001724983, 2.5874744999999998e-05], "gradient_norm": 3.194444179534912, "seconds": 0.3126735910000207, "nonpadding_tokens_per_second": 347611.7047569675, "ranks": [{"compute_seconds": 0.3071744609999314, "nonpadding_tokens": 6914, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.3100789559999839, "nonpadding_tokens": 12546, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3067009029999781, "nonpadding_tokens": 29010, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.3103633619999755, "nonpadding_tokens": 7361, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30932894700004, "nonpadding_tokens": 6038, "peak_allocated_bytes": 14966105600, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.30573498599994764, "nonpadding_tokens": 14516, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.3085476800000606, "nonpadding_tokens": 10212, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.30661053899996205, "nonpadding_tokens": 22092, "peak_allocated_bytes": 14972038656, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003499959999544444, "prefetch": {"data_wait_seconds": 0.00014942800009976054, "consumed_batches": 14, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 19, "tokens": 1813216, "loss_tokens": 88234, "loss": 9.94035462520117, "lr": 2.7198254999999997e-05, "optimizer_lrs": [0.001813217, 2.7198254999999997e-05], "gradient_norm": 2.775146007537842, "seconds": 0.3108032979999962, "nonpadding_tokens_per_second": 283890.16644218843, "ranks": [{"compute_seconds": 0.3070093750000069, "nonpadding_tokens": 13821, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18092130304}, {"compute_seconds": 0.30516517999990356, "nonpadding_tokens": 12269, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.30843438900001274, "nonpadding_tokens": 7561, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.3047295659999918, "nonpadding_tokens": 7816, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3078800550000551, "nonpadding_tokens": 4318, "peak_allocated_bytes": 14966105600, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.3044677230000161, "nonpadding_tokens": 19622, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.30742959099995915, "nonpadding_tokens": 3018, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3082955149999407, "nonpadding_tokens": 19809, "peak_allocated_bytes": 14972038656, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003496020000284261, "prefetch": {"data_wait_seconds": 0.000158384000087608, "consumed_batches": 15, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 20, "tokens": 1930955, "loss_tokens": 117739, "loss": 9.8764820917453, "lr": 2.896434e-05, "optimizer_lrs": [0.0019309560000000002, 2.896434e-05], "gradient_norm": 2.6086273193359375, "seconds": 0.30698771200002284, "nonpadding_tokens_per_second": 383530.00917506183, "ranks": [{"compute_seconds": 0.26454907999993793, "nonpadding_tokens": 18741, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18094227456}, {"compute_seconds": 0.2706041279999454, "nonpadding_tokens": 18443, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.2708476639999162, "nonpadding_tokens": 11294, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.2751059180000084, "nonpadding_tokens": 17832, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2742764319999651, "nonpadding_tokens": 5789, "peak_allocated_bytes": 14966105600, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.2754907070000172, "nonpadding_tokens": 14522, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.27380985100001, "nonpadding_tokens": 17364, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.2728577259999838, "nonpadding_tokens": 13754, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.011671596000041973, "prefetch": {"data_wait_seconds": 0.0001683410000623553, "consumed_batches": 16, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 21, "tokens": 2016176, "loss_tokens": 85221, "loss": 9.780083547482429, "lr": 3.0242654999999998e-05, "optimizer_lrs": [0.002016177, 3.0242654999999998e-05], "gradient_norm": 2.5415115356445312, "seconds": 0.3849354100000255, "nonpadding_tokens_per_second": 221390.388584917, "ranks": [{"compute_seconds": 0.3801147230000197, "nonpadding_tokens": 17586, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18094227456}, {"compute_seconds": 0.3825791299999537, "nonpadding_tokens": 9468, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3792001809999874, "nonpadding_tokens": 5694, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.37826288899998417, "nonpadding_tokens": 4818, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.38199112699999205, "nonpadding_tokens": 4666, "peak_allocated_bytes": 14966105600, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.37841147399990405, "nonpadding_tokens": 11542, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.3814249260000224, "nonpadding_tokens": 11365, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3820392620000348, "nonpadding_tokens": 20082, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00037257999997564184, "prefetch": {"data_wait_seconds": 0.00017849100004241336, "consumed_batches": 17, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 22, "tokens": 2087134, "loss_tokens": 70958, "loss": 9.661119958285182, "lr": 3.1307025e-05, "optimizer_lrs": [0.002087135, 3.1307025e-05], "gradient_norm": 2.5243332386016846, "seconds": 0.28433577499993135, "nonpadding_tokens_per_second": 249557.05978263597, "ranks": [{"compute_seconds": 0.2806084290000399, "nonpadding_tokens": 6171, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18094227456}, {"compute_seconds": 0.28199513200002, "nonpadding_tokens": 13481, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.27807236499995724, "nonpadding_tokens": 6643, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.2819018909999613, "nonpadding_tokens": 6410, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.28037204999998266, "nonpadding_tokens": 6108, "peak_allocated_bytes": 14966105600, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.27769551699998374, "nonpadding_tokens": 13440, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19191037952}, {"compute_seconds": 0.28108670300002814, "nonpadding_tokens": 2865, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.28024584499996763, "nonpadding_tokens": 15840, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00035172299999430834, "prefetch": {"data_wait_seconds": 0.0001879640000197469, "consumed_batches": 18, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 23, "tokens": 2160362, "loss_tokens": 73228, "loss": 9.698520374720053, "lr": 3.2405444999999996e-05, "optimizer_lrs": [0.0021603629999999998, 3.2405444999999996e-05], "gradient_norm": 2.4286696910858154, "seconds": 0.34264164100000016, "nonpadding_tokens_per_second": 213715.9972333893, "ranks": [{"compute_seconds": 0.33752443199989557, "nonpadding_tokens": 10979, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18094227456}, {"compute_seconds": 0.3386886279999999, "nonpadding_tokens": 8284, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3354846610000095, "nonpadding_tokens": 14700, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.34027820900007555, "nonpadding_tokens": 7144, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3358071799999607, "nonpadding_tokens": 4668, "peak_allocated_bytes": 14966105600, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.33501858299996456, "nonpadding_tokens": 7999, "peak_allocated_bytes": 14969897472, "peak_reserved_bytes": 19193135104}, {"compute_seconds": 0.3365152979999948, "nonpadding_tokens": 3131, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.33810726100000466, "nonpadding_tokens": 16323, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00038973000005171343, "prefetch": {"data_wait_seconds": 0.00019730999997591425, "consumed_batches": 19, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 24, "tokens": 2234444, "loss_tokens": 74082, "loss": 9.539562579304015, "lr": 3.3516674999999995e-05, "optimizer_lrs": [0.002234445, 3.3516674999999995e-05], "gradient_norm": 3.927574634552002, "seconds": 0.37569844099994043, "nonpadding_tokens_per_second": 197184.74157845054, "ranks": [{"compute_seconds": 0.3731571499999973, "nonpadding_tokens": 18259, "peak_allocated_bytes": 14996222464, "peak_reserved_bytes": 18094227456}, {"compute_seconds": 0.3729190880000033, "nonpadding_tokens": 8927, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3704794049999691, "nonpadding_tokens": 6769, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.37332480299994586, "nonpadding_tokens": 11520, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37208412400002544, "nonpadding_tokens": 3555, "peak_allocated_bytes": 14966105600, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.36990967599990654, "nonpadding_tokens": 14664, "peak_allocated_bytes": 14974920192, "peak_reserved_bytes": 19193135104}, {"compute_seconds": 0.3732926860000134, "nonpadding_tokens": 7419, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3729671879999614, "nonpadding_tokens": 2969, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00037043899999389396, "prefetch": {"data_wait_seconds": 0.00020722800002204167, "consumed_batches": 20, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 25, "tokens": 2334271, "loss_tokens": 99827, "loss": 9.714461643643503, "lr": 3.501408e-05, "optimizer_lrs": [0.002334272, 3.501408e-05], "gradient_norm": 2.3812623023986816, "seconds": 0.35037740100005976, "nonpadding_tokens_per_second": 284912.7818034787, "ranks": [{"compute_seconds": 0.34536780999997063, "nonpadding_tokens": 14430, "peak_allocated_bytes": 14997406208, "peak_reserved_bytes": 18094227456}, {"compute_seconds": 0.34648704199992153, "nonpadding_tokens": 14223, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3436222569999927, "nonpadding_tokens": 10941, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.34337499299999763, "nonpadding_tokens": 17597, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3457581699999537, "nonpadding_tokens": 17593, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.3479464490000055, "nonpadding_tokens": 12581, "peak_allocated_bytes": 14974920192, "peak_reserved_bytes": 19193135104}, {"compute_seconds": 0.3467571720000251, "nonpadding_tokens": 5444, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3462585529999842, "nonpadding_tokens": 7018, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00036212300005900033, "prefetch": {"data_wait_seconds": 0.00021728100000473205, "consumed_batches": 21, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 26, "tokens": 2398757, "loss_tokens": 64486, "loss": 9.581273454703346, "lr": 3.598137e-05, "optimizer_lrs": [0.002398758, 3.598137e-05], "gradient_norm": 2.0718424320220947, "seconds": 0.6251303500000631, "nonpadding_tokens_per_second": 103156.08576674206, "ranks": [{"compute_seconds": 0.6202860699999064, "nonpadding_tokens": 7495, "peak_allocated_bytes": 14997406208, "peak_reserved_bytes": 18094227456}, {"compute_seconds": 0.6226108079999904, "nonpadding_tokens": 4353, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.6217015099999799, "nonpadding_tokens": 4150, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.6224809499999537, "nonpadding_tokens": 13645, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.6199846409999736, "nonpadding_tokens": 6657, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.6185134250000601, "nonpadding_tokens": 20729, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19193135104}, {"compute_seconds": 0.6202350220000881, "nonpadding_tokens": 5304, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.6215870009999662, "nonpadding_tokens": 2153, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003509069999836356, "prefetch": {"data_wait_seconds": 0.00022649799996088404, "consumed_batches": 22, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 27, "tokens": 2488829, "loss_tokens": 90072, "loss": 9.529298089306332, "lr": 3.733245e-05, "optimizer_lrs": [0.00248883, 3.733245e-05], "gradient_norm": 2.3529539108276367, "seconds": 0.45765994700002466, "nonpadding_tokens_per_second": 196809.88163903088, "ranks": [{"compute_seconds": 0.4509357120000459, "nonpadding_tokens": 14069, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18094227456}, {"compute_seconds": 0.45422365599995373, "nonpadding_tokens": 14121, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.45326784199994563, "nonpadding_tokens": 7438, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20166213632}, {"compute_seconds": 0.4546860650000326, "nonpadding_tokens": 8120, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4537170320000996, "nonpadding_tokens": 3026, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.45522844300001, "nonpadding_tokens": 20889, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19193135104}, {"compute_seconds": 0.4536941400000387, "nonpadding_tokens": 5981, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.45356544399999166, "nonpadding_tokens": 16428, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.000354064000021026, "prefetch": {"data_wait_seconds": 0.0002360189998853457, "consumed_batches": 23, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 28, "tokens": 2568571, "loss_tokens": 79742, "loss": 9.47917189185122, "lr": 3.852858e-05, "optimizer_lrs": [0.0025685720000000003, 3.852858e-05], "gradient_norm": 2.0785233974456787, "seconds": 0.3557860129999426, "nonpadding_tokens_per_second": 224129.10313034948, "ranks": [{"compute_seconds": 0.35269798800004537, "nonpadding_tokens": 9579, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18094227456}, {"compute_seconds": 0.3524282519999815, "nonpadding_tokens": 19494, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.34926935999999387, "nonpadding_tokens": 11255, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35348225800009914, "nonpadding_tokens": 2412, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3523267760000408, "nonpadding_tokens": 13359, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.34931703500001277, "nonpadding_tokens": 5653, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19193135104}, {"compute_seconds": 0.3503108559999646, "nonpadding_tokens": 10851, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3501339460000281, "nonpadding_tokens": 7139, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00035672100000283535, "prefetch": {"data_wait_seconds": 0.00024493199987318803, "consumed_batches": 24, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 29, "tokens": 2654299, "loss_tokens": 85728, "loss": 9.399688549832026, "lr": 3.9814499999999996e-05, "optimizer_lrs": [0.0026543, 3.9814499999999996e-05], "gradient_norm": 1.8458232879638672, "seconds": 0.347091665999983, "nonpadding_tokens_per_second": 246989.50852943904, "ranks": [{"compute_seconds": 0.34198939899999914, "nonpadding_tokens": 4101, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18115198976}, {"compute_seconds": 0.34449992500003646, "nonpadding_tokens": 13656, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3401521480000156, "nonpadding_tokens": 3381, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3447514010000532, "nonpadding_tokens": 19514, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.342461602999947, "nonpadding_tokens": 9160, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19228786688}, {"compute_seconds": 0.3403592409999874, "nonpadding_tokens": 10338, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19193135104}, {"compute_seconds": 0.34113934399999835, "nonpadding_tokens": 16018, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.34346652800002175, "nonpadding_tokens": 9560, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003686539999989691, "prefetch": {"data_wait_seconds": 0.0002540589998716314, "consumed_batches": 25, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 30, "tokens": 2776179, "loss_tokens": 121880, "loss": 9.393493600262554, "lr": 4.1642699999999993e-05, "optimizer_lrs": [0.0027761799999999996, 4.1642699999999993e-05], "gradient_norm": 2.095242500305176, "seconds": 0.3149553119999382, "nonpadding_tokens_per_second": 386975.53384977963, "ranks": [{"compute_seconds": 0.3096345019999944, "nonpadding_tokens": 10778, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18115198976}, {"compute_seconds": 0.3083442129999412, "nonpadding_tokens": 12535, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.311557328000049, "nonpadding_tokens": 13840, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3127110420000463, "nonpadding_tokens": 18891, "peak_allocated_bytes": 14970927616, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31037467500004823, "nonpadding_tokens": 2648, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.308708202000048, "nonpadding_tokens": 18767, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19193135104}, {"compute_seconds": 0.3116963319999968, "nonpadding_tokens": 25367, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3091372169999431, "nonpadding_tokens": 19054, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00036158199998226337, "prefetch": {"data_wait_seconds": 0.0002632379998885881, "consumed_batches": 26, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 31, "tokens": 2906087, "loss_tokens": 129908, "loss": 9.316072720694645, "lr": 4.359132e-05, "optimizer_lrs": [0.0029060880000000002, 4.359132e-05], "gradient_norm": 1.9508576393127441, "seconds": 0.33435535099999925, "nonpadding_tokens_per_second": 388532.7380329567, "ranks": [{"compute_seconds": 0.3199473320000834, "nonpadding_tokens": 13938, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18115198976}, {"compute_seconds": 0.31961446900004375, "nonpadding_tokens": 27944, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3204223660000025, "nonpadding_tokens": 19476, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3241972999999234, "nonpadding_tokens": 10495, "peak_allocated_bytes": 14982979072, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3228914310000164, "nonpadding_tokens": 27495, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3200982060000115, "nonpadding_tokens": 17427, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19193135104}, {"compute_seconds": 0.32223313899999084, "nonpadding_tokens": 8697, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.32062594899991836, "nonpadding_tokens": 4436, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00037973099995269877, "prefetch": {"data_wait_seconds": 0.0002717269998129268, "consumed_batches": 27, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 32, "tokens": 3027368, "loss_tokens": 121281, "loss": 9.080044483472268, "lr": 4.541053499999999e-05, "optimizer_lrs": [0.0030273689999999994, 4.541053499999999e-05], "gradient_norm": 2.1934196949005127, "seconds": 0.3257758480000348, "nonpadding_tokens_per_second": 372283.58315864793, "ranks": [{"compute_seconds": 0.3188946909999686, "nonpadding_tokens": 12670, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18115198976}, {"compute_seconds": 0.3206531459999269, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.32346700800007966, "nonpadding_tokens": 12305, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3164891730000363, "nonpadding_tokens": 9118, "peak_allocated_bytes": 14982979072, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3192424599999413, "nonpadding_tokens": 28956, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.316943910999953, "nonpadding_tokens": 4780, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19193135104}, {"compute_seconds": 0.32004955299998983, "nonpadding_tokens": 10475, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.31952517199999875, "nonpadding_tokens": 10209, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003551790000528854, "prefetch": {"data_wait_seconds": 0.000279528999840295, "consumed_batches": 28, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 33, "tokens": 3103179, "loss_tokens": 75811, "loss": 9.15470957380855, "lr": 4.6547699999999995e-05, "optimizer_lrs": [0.00310318, 4.6547699999999995e-05], "gradient_norm": 1.9179601669311523, "seconds": 0.32464692900009595, "nonpadding_tokens_per_second": 233518.3032024849, "ranks": [{"compute_seconds": 0.3208837529999755, "nonpadding_tokens": 9641, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18115198976}, {"compute_seconds": 0.3174477119999892, "nonpadding_tokens": 9424, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3214956220000431, "nonpadding_tokens": 13665, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32227991599995676, "nonpadding_tokens": 4911, "peak_allocated_bytes": 14982979072, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32082612099998187, "nonpadding_tokens": 9421, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3174160830000119, "nonpadding_tokens": 2418, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3198817689999487, "nonpadding_tokens": 8640, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3198299439999346, "nonpadding_tokens": 17691, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003538180000077773, "prefetch": {"data_wait_seconds": 0.00028808499985188973, "consumed_batches": 29, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 34, "tokens": 3176127, "loss_tokens": 72948, "loss": 9.168852298897844, "lr": 4.764191999999999e-05, "optimizer_lrs": [0.0031761279999999994, 4.764191999999999e-05], "gradient_norm": 1.7893275022506714, "seconds": 0.3521021589999691, "nonpadding_tokens_per_second": 207178.5080988566, "ranks": [{"compute_seconds": 0.34741407599995, "nonpadding_tokens": 17387, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18115198976}, {"compute_seconds": 0.34887754700002915, "nonpadding_tokens": 7660, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3452686289999747, "nonpadding_tokens": 7689, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34896718100003454, "nonpadding_tokens": 13423, "peak_allocated_bytes": 14982979072, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34490671499997916, "nonpadding_tokens": 3161, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.34967806599991036, "nonpadding_tokens": 9035, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.347909876000017, "nonpadding_tokens": 4093, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.34534986999995, "nonpadding_tokens": 10500, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0004122280000729006, "prefetch": {"data_wait_seconds": 0.0002992709997897691, "consumed_batches": 30, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 35, "tokens": 3255440, "loss_tokens": 79313, "loss": 9.131453860022948, "lr": 4.8831614999999996e-05, "optimizer_lrs": [0.003255441, 4.8831614999999996e-05], "gradient_norm": 1.9646848440170288, "seconds": 0.38703222000003734, "nonpadding_tokens_per_second": 204926.09116624025, "ranks": [{"compute_seconds": 0.3823184149999861, "nonpadding_tokens": 11798, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18115198976}, {"compute_seconds": 0.3846051699999862, "nonpadding_tokens": 8150, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19648217088}, {"compute_seconds": 0.3818636460000562, "nonpadding_tokens": 9212, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3817928839999922, "nonpadding_tokens": 10371, "peak_allocated_bytes": 14982979072, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.381680938000045, "nonpadding_tokens": 13837, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.38088188400001854, "nonpadding_tokens": 11203, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.38181948199996896, "nonpadding_tokens": 11251, "peak_allocated_bytes": 14981652992, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3817195609999544, "nonpadding_tokens": 3491, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0004223600000159422, "prefetch": {"data_wait_seconds": 0.00031011599980956817, "consumed_batches": 31, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 36, "tokens": 3337859, "loss_tokens": 82419, "loss": 9.07132153993618, "lr": 5.00679e-05, "optimizer_lrs": [0.0033378600000000002, 5.00679e-05], "gradient_norm": 1.7048972845077515, "seconds": 0.4848028210000166, "nonpadding_tokens_per_second": 170005.19887650813, "ranks": [{"compute_seconds": 0.44099579499993524, "nonpadding_tokens": 9620, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18115198976}, {"compute_seconds": 0.4790156039999829, "nonpadding_tokens": 1857, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.47501599099996383, "nonpadding_tokens": 4395, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.47470318599994243, "nonpadding_tokens": 9379, "peak_allocated_bytes": 14982979072, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.47669378699993104, "nonpadding_tokens": 10102, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.47964720599998145, "nonpadding_tokens": 16862, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4752549469999394, "nonpadding_tokens": 10639, "peak_allocated_bytes": 14983090688, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.4754123660000005, "nonpadding_tokens": 19565, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00038724299997738854, "prefetch": {"data_wait_seconds": 0.00031939199982389255, "consumed_batches": 32, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 37, "tokens": 3421480, "loss_tokens": 83621, "loss": 9.00742262709128, "lr": 5.1322214999999996e-05, "optimizer_lrs": [0.003421481, 5.1322214999999996e-05], "gradient_norm": 1.7235795259475708, "seconds": 0.3123424430000341, "nonpadding_tokens_per_second": 267722.1808116256, "ranks": [{"compute_seconds": 0.30861540000000787, "nonpadding_tokens": 2405, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.3099856579999596, "nonpadding_tokens": 13219, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30631949700000405, "nonpadding_tokens": 12343, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30517796799995267, "nonpadding_tokens": 8729, "peak_allocated_bytes": 14982979072, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30958402499993554, "nonpadding_tokens": 12327, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.30597194999995736, "nonpadding_tokens": 11152, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.30905619600002865, "nonpadding_tokens": 10508, "peak_allocated_bytes": 14983090688, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.30899187500006065, "nonpadding_tokens": 12938, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003551590000370197, "prefetch": {"data_wait_seconds": 0.00032759199984866427, "consumed_batches": 33, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 38, "tokens": 3495202, "loss_tokens": 73722, "loss": 9.016653780418329, "lr": 5.2428044999999997e-05, "optimizer_lrs": [0.003495203, 5.2428044999999997e-05], "gradient_norm": 1.6833306550979614, "seconds": 0.3144921920001025, "nonpadding_tokens_per_second": 234415.9946583856, "ranks": [{"compute_seconds": 0.3086968090000255, "nonpadding_tokens": 4542, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.3112850329999901, "nonpadding_tokens": 12682, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30740128299999014, "nonpadding_tokens": 18091, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30829729300000963, "nonpadding_tokens": 4961, "peak_allocated_bytes": 14982979072, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3120488199999727, "nonpadding_tokens": 10898, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3108180540000376, "nonpadding_tokens": 5138, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31022002299994256, "nonpadding_tokens": 12697, "peak_allocated_bytes": 14983090688, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3105549260000089, "nonpadding_tokens": 4713, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00037125099993318145, "prefetch": {"data_wait_seconds": 0.0003364969999211098, "consumed_batches": 34, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 39, "tokens": 3606580, "loss_tokens": 111378, "loss": 8.875694145163317, "lr": 5.409871499999999e-05, "optimizer_lrs": [0.0036065809999999998, 5.409871499999999e-05], "gradient_norm": 2.0706543922424316, "seconds": 0.30160837999994783, "nonpadding_tokens_per_second": 369280.1904244811, "ranks": [{"compute_seconds": 0.2973496049999085, "nonpadding_tokens": 13963, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.2992635919999884, "nonpadding_tokens": 6152, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2970713219999652, "nonpadding_tokens": 5993, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.29862867800000004, "nonpadding_tokens": 18449, "peak_allocated_bytes": 14982979072, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2979717499999879, "nonpadding_tokens": 18411, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.2957019149999951, "nonpadding_tokens": 6249, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2977652729999818, "nonpadding_tokens": 27778, "peak_allocated_bytes": 14983090688, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.2974995700000136, "nonpadding_tokens": 14383, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00037631099996815465, "prefetch": {"data_wait_seconds": 0.0003455679999433414, "consumed_batches": 35, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 40, "tokens": 3696735, "loss_tokens": 90155, "loss": 8.895316399534135, "lr": 5.545103999999999e-05, "optimizer_lrs": [0.0036967359999999995, 5.545103999999999e-05], "gradient_norm": 1.5873886346817017, "seconds": 0.3049478850000469, "nonpadding_tokens_per_second": 295640.67971806444, "ranks": [{"compute_seconds": 0.29799520899996423, "nonpadding_tokens": 17619, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.3023282479999807, "nonpadding_tokens": 10846, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2985869020000109, "nonpadding_tokens": 15125, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3013115869999865, "nonpadding_tokens": 11530, "peak_allocated_bytes": 14984264192, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.29936464899992643, "nonpadding_tokens": 3967, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.30251074899990726, "nonpadding_tokens": 7270, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3010782520000248, "nonpadding_tokens": 15727, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.2990538579999793, "nonpadding_tokens": 8071, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00038682799993239314, "prefetch": {"data_wait_seconds": 0.00035485699993387243, "consumed_batches": 36, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 41, "tokens": 3810866, "loss_tokens": 114131, "loss": 8.873593173633807, "lr": 5.7163004999999995e-05, "optimizer_lrs": [0.003810867, 5.7163004999999995e-05], "gradient_norm": 1.738876461982727, "seconds": 0.5364733650000062, "nonpadding_tokens_per_second": 212743.08744106742, "ranks": [{"compute_seconds": 0.5334945090000929, "nonpadding_tokens": 25166, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.5340629219999755, "nonpadding_tokens": 9581, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.5331710400000702, "nonpadding_tokens": 9214, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.529893027000071, "nonpadding_tokens": 15654, "peak_allocated_bytes": 14984264192, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.5307380409999496, "nonpadding_tokens": 21667, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.5301170200000342, "nonpadding_tokens": 7655, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.5309783399999333, "nonpadding_tokens": 14050, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.5308208640000203, "nonpadding_tokens": 11144, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00040022999996836006, "prefetch": {"data_wait_seconds": 0.00036520799994832487, "consumed_batches": 37, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 42, "tokens": 3926963, "loss_tokens": 116097, "loss": 8.816086440648768, "lr": 5.8904459999999995e-05, "optimizer_lrs": [0.003926964, 5.8904459999999995e-05], "gradient_norm": 1.7717775106430054, "seconds": 0.2827258969999775, "nonpadding_tokens_per_second": 410634.47399729793, "ranks": [{"compute_seconds": 0.2771148289999701, "nonpadding_tokens": 8339, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.28007595099995797, "nonpadding_tokens": 13025, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.27681408699993426, "nonpadding_tokens": 14464, "peak_allocated_bytes": 14976231936, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.280318450999971, "nonpadding_tokens": 17532, "peak_allocated_bytes": 14984264192, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.27654182200001287, "nonpadding_tokens": 2429, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.2758865760000617, "nonpadding_tokens": 23886, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.27649639200001275, "nonpadding_tokens": 16129, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.278743643000098, "nonpadding_tokens": 20293, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003837089999478849, "prefetch": {"data_wait_seconds": 0.0003745239999943806, "consumed_batches": 38, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 43, "tokens": 4020875, "loss_tokens": 93912, "loss": 8.72744031646648, "lr": 6.031313999999999e-05, "optimizer_lrs": [0.004020876, 6.031313999999999e-05], "gradient_norm": 1.6587564945220947, "seconds": 0.3189422299999478, "nonpadding_tokens_per_second": 294448.30808392906, "ranks": [{"compute_seconds": 0.3159094399999276, "nonpadding_tokens": 4011, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.31636498600005325, "nonpadding_tokens": 13013, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3125310890000037, "nonpadding_tokens": 23003, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3166224349999993, "nonpadding_tokens": 9759, "peak_allocated_bytes": 14984264192, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31360323000001245, "nonpadding_tokens": 7387, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3122114349999947, "nonpadding_tokens": 15612, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3123789139999644, "nonpadding_tokens": 17206, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.31535194399998545, "nonpadding_tokens": 3921, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00034061500002735556, "prefetch": {"data_wait_seconds": 0.0003825509999160204, "consumed_batches": 39, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 44, "tokens": 4108598, "loss_tokens": 87723, "loss": 8.798748617808329, "lr": 6.1628985e-05, "optimizer_lrs": [0.004108599, 6.1628985e-05], "gradient_norm": 1.5874465703964233, "seconds": 0.38657020299990563, "nonpadding_tokens_per_second": 226926.4400598962, "ranks": [{"compute_seconds": 0.38165227199999663, "nonpadding_tokens": 11319, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.37879346499994426, "nonpadding_tokens": 8544, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.38021237700002075, "nonpadding_tokens": 12108, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38414688000000297, "nonpadding_tokens": 13444, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37959426700001586, "nonpadding_tokens": 3618, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.37976819599998635, "nonpadding_tokens": 18019, "peak_allocated_bytes": 14977491456, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3808340200000657, "nonpadding_tokens": 11438, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3831934539999793, "nonpadding_tokens": 9233, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0004053519999160926, "prefetch": {"data_wait_seconds": 0.0003922610000017812, "consumed_batches": 40, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 45, "tokens": 4254154, "loss_tokens": 145556, "loss": 8.608801767017505, "lr": 6.3812325e-05, "optimizer_lrs": [0.004254155, 6.3812325e-05], "gradient_norm": 1.887671947479248, "seconds": 0.3433502650000264, "nonpadding_tokens_per_second": 423928.6082973864, "ranks": [{"compute_seconds": 0.34092638200002057, "nonpadding_tokens": 23088, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.3360577949999879, "nonpadding_tokens": 11208, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33935099400002855, "nonpadding_tokens": 18632, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3409686600000441, "nonpadding_tokens": 24921, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33953027200004726, "nonpadding_tokens": 9217, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.33616591300005894, "nonpadding_tokens": 18924, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3395850580000115, "nonpadding_tokens": 21416, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.33931992200007244, "nonpadding_tokens": 18150, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00034554999990632496, "prefetch": {"data_wait_seconds": 0.0004007120001006115, "consumed_batches": 41, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 46, "tokens": 4375007, "loss_tokens": 120853, "loss": 8.631079700131565, "lr": 6.562512e-05, "optimizer_lrs": [0.0043750080000000005, 6.562512e-05], "gradient_norm": 1.8809030055999756, "seconds": 0.3644133000000238, "nonpadding_tokens_per_second": 331637.18228723295, "ranks": [{"compute_seconds": 0.3594203940000398, "nonpadding_tokens": 18825, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.3620509450000782, "nonpadding_tokens": 16983, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35866940500000055, "nonpadding_tokens": 9881, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35719590799999423, "nonpadding_tokens": 18537, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36095719100001133, "nonpadding_tokens": 26227, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.35819994500002394, "nonpadding_tokens": 8780, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3612474409999322, "nonpadding_tokens": 7009, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.35735159299997576, "nonpadding_tokens": 14611, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003915800000413583, "prefetch": {"data_wait_seconds": 0.0004100790000620691, "consumed_batches": 42, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 47, "tokens": 4465931, "loss_tokens": 90924, "loss": 8.79245578725089, "lr": 6.698898e-05, "optimizer_lrs": [0.004465932000000001, 6.698898e-05], "gradient_norm": 1.5249569416046143, "seconds": 0.40460737099999733, "nonpadding_tokens_per_second": 224721.560992028, "ranks": [{"compute_seconds": 0.36979237599996395, "nonpadding_tokens": 20373, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.3723467190000065, "nonpadding_tokens": 10494, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.39938515400001506, "nonpadding_tokens": 5303, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3736662310000156, "nonpadding_tokens": 10351, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.40145945599999777, "nonpadding_tokens": 4783, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.37375319700004184, "nonpadding_tokens": 20898, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.37041365300001416, "nonpadding_tokens": 6003, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3733288390000098, "nonpadding_tokens": 12719, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003646599999456157, "prefetch": {"data_wait_seconds": 0.0004189460000816325, "consumed_batches": 43, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 48, "tokens": 4560289, "loss_tokens": 94358, "loss": 8.527292333453444, "lr": 6.840435e-05, "optimizer_lrs": [0.004560290000000001, 6.840435e-05], "gradient_norm": 1.832521915435791, "seconds": 0.3394984370000884, "nonpadding_tokens_per_second": 277933.53287212754, "ranks": [{"compute_seconds": 0.3332527699999446, "nonpadding_tokens": 9865, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.337117070999966, "nonpadding_tokens": 19353, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33465450799997143, "nonpadding_tokens": 6252, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3368778750000274, "nonpadding_tokens": 4977, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3352144689999932, "nonpadding_tokens": 5356, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.33211176899999373, "nonpadding_tokens": 6117, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3352756849999423, "nonpadding_tokens": 25208, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.33503581000002214, "nonpadding_tokens": 17230, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00039600399998107605, "prefetch": {"data_wait_seconds": 0.0004284990001224287, "consumed_batches": 44, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 49, "tokens": 4660509, "loss_tokens": 100220, "loss": 8.429391588505288, "lr": 6.990765e-05, "optimizer_lrs": [0.0046605100000000005, 6.990765e-05], "gradient_norm": 2.2305550575256348, "seconds": 0.35177255199994306, "nonpadding_tokens_per_second": 284899.9998158362, "ranks": [{"compute_seconds": 0.3482755999999654, "nonpadding_tokens": 14007, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.34921322999991844, "nonpadding_tokens": 13478, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34609612899998865, "nonpadding_tokens": 3851, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3494767539999657, "nonpadding_tokens": 20210, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3455924429999868, "nonpadding_tokens": 3420, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3452632690000428, "nonpadding_tokens": 12925, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3459797680000065, "nonpadding_tokens": 17689, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.34814164699992034, "nonpadding_tokens": 14640, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003786350000609673, "prefetch": {"data_wait_seconds": 0.0004372560001684178, "consumed_batches": 45, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 50, "tokens": 4710420, "loss_tokens": 49911, "loss": 8.601914532868506, "lr": 7.0656315e-05, "optimizer_lrs": [0.004710421, 7.0656315e-05], "gradient_norm": 2.1341552734375, "seconds": 0.2642517689999977, "nonpadding_tokens_per_second": 188876.69206104893, "ranks": [{"compute_seconds": 0.2587803270000677, "nonpadding_tokens": 3287, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.2615466510000033, "nonpadding_tokens": 2608, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.26042833700000756, "nonpadding_tokens": 8152, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2619521139999961, "nonpadding_tokens": 2656, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.25968886999999086, "nonpadding_tokens": 12551, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.25758893900001567, "nonpadding_tokens": 4344, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.261099339999987, "nonpadding_tokens": 3884, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.2578594190000558, "nonpadding_tokens": 12429, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00037508999992041936, "prefetch": {"data_wait_seconds": 0.00044609200017475814, "consumed_batches": 46, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 51, "tokens": 4782522, "loss_tokens": 72102, "loss": 8.424343291448226, "lr": 7.173784499999999e-05, "optimizer_lrs": [0.0047825229999999995, 7.173784499999999e-05], "gradient_norm": 1.8104947805404663, "seconds": 0.2849516539999968, "nonpadding_tokens_per_second": 253032.39685705004, "ranks": [{"compute_seconds": 0.28010651699992195, "nonpadding_tokens": 6793, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.2702797140000257, "nonpadding_tokens": 9726, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2723214510000389, "nonpadding_tokens": 8466, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.28264756199996555, "nonpadding_tokens": 6562, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.27130163900005755, "nonpadding_tokens": 13238, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.2699775260000479, "nonpadding_tokens": 12968, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2729271959999551, "nonpadding_tokens": 12439, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.27371975799997017, "nonpadding_tokens": 1910, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003812479999396601, "prefetch": {"data_wait_seconds": 0.00045517400019434717, "consumed_batches": 47, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 52, "tokens": 4897049, "loss_tokens": 114527, "loss": 8.272564548097828, "lr": 7.345574999999999e-05, "optimizer_lrs": [0.00489705, 7.345574999999999e-05], "gradient_norm": 2.359999179840088, "seconds": 0.3416676719999714, "nonpadding_tokens_per_second": 335199.98930425465, "ranks": [{"compute_seconds": 0.33711681600004795, "nonpadding_tokens": 10588, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.33821274100000664, "nonpadding_tokens": 19793, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3357165789999499, "nonpadding_tokens": 19003, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3394044550000217, "nonpadding_tokens": 11382, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3385588439999765, "nonpadding_tokens": 8621, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3354871500000627, "nonpadding_tokens": 13102, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33825156600005357, "nonpadding_tokens": 12130, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.33812211999998, "nonpadding_tokens": 19908, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00034701899994615815, "prefetch": {"data_wait_seconds": 0.0004644330001610797, "consumed_batches": 48, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 53, "tokens": 4988372, "loss_tokens": 91323, "loss": 8.333659100117167, "lr": 7.4825595e-05, "optimizer_lrs": [0.004988373, 7.4825595e-05], "gradient_norm": 2.0589494705200195, "seconds": 0.3144947590000129, "nonpadding_tokens_per_second": 290380.03777988633, "ranks": [{"compute_seconds": 0.31207729800007655, "nonpadding_tokens": 11406, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.31173268100008045, "nonpadding_tokens": 4852, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30897779500003253, "nonpadding_tokens": 9269, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31181062699999984, "nonpadding_tokens": 20024, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3097521729999926, "nonpadding_tokens": 12574, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3085737149999659, "nonpadding_tokens": 11790, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3117523500000061, "nonpadding_tokens": 9435, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3098400829999264, "nonpadding_tokens": 11973, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00037269099993864074, "prefetch": {"data_wait_seconds": 0.0004742830001305265, "consumed_batches": 49, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 54, "tokens": 5043736, "loss_tokens": 55364, "loss": 8.193348227187341, "lr": 7.565605499999999e-05, "optimizer_lrs": [0.0050437369999999995, 7.565605499999999e-05], "gradient_norm": 2.0297133922576904, "seconds": 0.32771903299999394, "nonpadding_tokens_per_second": 168937.3958332198, "ranks": [{"compute_seconds": 0.32449435099999846, "nonpadding_tokens": 6629, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.3247581829999717, "nonpadding_tokens": 7982, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32195405100003427, "nonpadding_tokens": 2740, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3210852330000762, "nonpadding_tokens": 5414, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32455779499991877, "nonpadding_tokens": 5977, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3215767259999893, "nonpadding_tokens": 10551, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32401837999998406, "nonpadding_tokens": 3692, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.32446171699996285, "nonpadding_tokens": 12379, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003625059999876612, "prefetch": {"data_wait_seconds": 0.00048406900020836474, "consumed_batches": 50, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 55, "tokens": 5108514, "loss_tokens": 64778, "loss": 8.128816882274846, "lr": 7.6627725e-05, "optimizer_lrs": [0.005108515, 7.6627725e-05], "gradient_norm": 2.489671468734741, "seconds": 0.3043230539999513, "nonpadding_tokens_per_second": 212859.32547197153, "ranks": [{"compute_seconds": 0.3001150699999471, "nonpadding_tokens": 7278, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.30116005100001075, "nonpadding_tokens": 9004, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.29770301999997173, "nonpadding_tokens": 16900, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3019572400000925, "nonpadding_tokens": 9241, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3002894319999996, "nonpadding_tokens": 3470, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.2976423720000412, "nonpadding_tokens": 2753, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3003121250000049, "nonpadding_tokens": 10368, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.30048808799995186, "nonpadding_tokens": 5764, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003607190000138871, "prefetch": {"data_wait_seconds": 0.0004952720003075228, "consumed_batches": 51, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 56, "tokens": 5185052, "loss_tokens": 76538, "loss": 8.143666381405316, "lr": 7.777579499999999e-05, "optimizer_lrs": [0.005185052999999999, 7.777579499999999e-05], "gradient_norm": 2.177682876586914, "seconds": 0.3587486149999677, "nonpadding_tokens_per_second": 213347.1651172978, "ranks": [{"compute_seconds": 0.35393066200003886, "nonpadding_tokens": 13199, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.3561180169999716, "nonpadding_tokens": 7618, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3556703959999368, "nonpadding_tokens": 9448, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35646066900005735, "nonpadding_tokens": 16913, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3532962779999025, "nonpadding_tokens": 9108, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3530065620000187, "nonpadding_tokens": 2700, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.35607828799993513, "nonpadding_tokens": 6243, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.35451965400000063, "nonpadding_tokens": 11309, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00036757300006229343, "prefetch": {"data_wait_seconds": 0.0005043750003324021, "consumed_batches": 52, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 57, "tokens": 5243807, "loss_tokens": 58755, "loss": 7.921865692281508, "lr": 7.865711999999998e-05, "optimizer_lrs": [0.005243807999999999, 7.865711999999998e-05], "gradient_norm": 2.321354627609253, "seconds": 0.2726050919999352, "nonpadding_tokens_per_second": 215531.55727558443, "ranks": [{"compute_seconds": 0.26789705800001684, "nonpadding_tokens": 5041, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18131976192}, {"compute_seconds": 0.2702644329999657, "nonpadding_tokens": 7364, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.26821921600003407, "nonpadding_tokens": 520, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.265138347000061, "nonpadding_tokens": 7495, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.26846898600001623, "nonpadding_tokens": 8967, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.2661718809999911, "nonpadding_tokens": 9896, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2664214640000182, "nonpadding_tokens": 11743, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.26919357399992805, "nonpadding_tokens": 7729, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00038157500000579603, "prefetch": {"data_wait_seconds": 0.0005149150002807801, "consumed_batches": 53, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 58, "tokens": 5329272, "loss_tokens": 85465, "loss": 7.987230883987597, "lr": 7.9939095e-05, "optimizer_lrs": [0.005329273, 7.9939095e-05], "gradient_norm": 2.3014824390411377, "seconds": 0.3092079879999119, "nonpadding_tokens_per_second": 276399.7157797371, "ranks": [{"compute_seconds": 0.3064437430000453, "nonpadding_tokens": 1529, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3035647390000804, "nonpadding_tokens": 7008, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30633177899994735, "nonpadding_tokens": 7071, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30690987300010875, "nonpadding_tokens": 12586, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30560333699997955, "nonpadding_tokens": 7346, "peak_allocated_bytes": 14967410176, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3030218569999761, "nonpadding_tokens": 9609, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.306021783999995, "nonpadding_tokens": 7548, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.30562611400000605, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003860250000116139, "prefetch": {"data_wait_seconds": 0.0005236180002157198, "consumed_batches": 54, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 59, "tokens": 5438583, "loss_tokens": 109311, "loss": 7.732347384984128, "lr": 8.157875999999998e-05, "optimizer_lrs": [0.005438583999999999, 8.157875999999998e-05], "gradient_norm": 2.6193273067474365, "seconds": 0.30231191099994703, "nonpadding_tokens_per_second": 361583.50373439025, "ranks": [{"compute_seconds": 0.29922493999993094, "nonpadding_tokens": 10275, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2999859059999608, "nonpadding_tokens": 11625, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2986320219999925, "nonpadding_tokens": 17797, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.29996059500001593, "nonpadding_tokens": 11862, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2988834320000251, "nonpadding_tokens": 9264, "peak_allocated_bytes": 14970580480, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.2959983570000304, "nonpadding_tokens": 17900, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2979762469999514, "nonpadding_tokens": 23094, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.2967153680000365, "nonpadding_tokens": 7494, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00038299099992400443, "prefetch": {"data_wait_seconds": 0.0005349490002117818, "consumed_batches": 55, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 60, "tokens": 5542966, "loss_tokens": 104383, "loss": 7.77087564545951, "lr": 8.314450499999999e-05, "optimizer_lrs": [0.005542966999999999, 8.314450499999999e-05], "gradient_norm": 2.316999912261963, "seconds": 0.34785634700006085, "nonpadding_tokens_per_second": 300075.0191859565, "ranks": [{"compute_seconds": 0.34493155699999534, "nonpadding_tokens": 5359, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3455638010000257, "nonpadding_tokens": 15774, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3426925819999269, "nonpadding_tokens": 17960, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3423020280000628, "nonpadding_tokens": 13020, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3421579739999743, "nonpadding_tokens": 2557, "peak_allocated_bytes": 14970580480, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.34231286200008526, "nonpadding_tokens": 9846, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3452957570000308, "nonpadding_tokens": 14177, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3455510100000083, "nonpadding_tokens": 25690, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003569090000610231, "prefetch": {"data_wait_seconds": 0.0005455400001892485, "consumed_batches": 56, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 61, "tokens": 5634528, "loss_tokens": 91562, "loss": 7.693713003210939, "lr": 8.4517935e-05, "optimizer_lrs": [0.005634529, 8.4517935e-05], "gradient_norm": 2.249972343444824, "seconds": 0.36507154399998853, "nonpadding_tokens_per_second": 250805.63386776284, "ranks": [{"compute_seconds": 0.36132586399992306, "nonpadding_tokens": 18968, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3617034359999707, "nonpadding_tokens": 7396, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35832662800009984, "nonpadding_tokens": 12855, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36231401999998525, "nonpadding_tokens": 6806, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36071986499996456, "nonpadding_tokens": 19599, "peak_allocated_bytes": 14970580480, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.357757797999966, "nonpadding_tokens": 11055, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3609190550000676, "nonpadding_tokens": 7727, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3590571260000388, "nonpadding_tokens": 7156, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00041373299995939306, "prefetch": {"data_wait_seconds": 0.0005571040002223526, "consumed_batches": 57, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 62, "tokens": 5707955, "loss_tokens": 73427, "loss": 7.580917101338745, "lr": 8.561933999999999e-05, "optimizer_lrs": [0.005707956, 8.561933999999999e-05], "gradient_norm": 2.0044169425964355, "seconds": 0.35945788999993056, "nonpadding_tokens_per_second": 204271.49338692825, "ranks": [{"compute_seconds": 0.3554652980000128, "nonpadding_tokens": 16465, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3534875310000416, "nonpadding_tokens": 6926, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35628611399999954, "nonpadding_tokens": 11877, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3571521430000075, "nonpadding_tokens": 2645, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35480309900003704, "nonpadding_tokens": 8147, "peak_allocated_bytes": 14970580480, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3542511340000374, "nonpadding_tokens": 6558, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3566170049999755, "nonpadding_tokens": 12197, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3562719990000005, "nonpadding_tokens": 8612, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003878219999933208, "prefetch": {"data_wait_seconds": 0.0005671780002103333, "consumed_batches": 58, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 63, "tokens": 5771254, "loss_tokens": 63299, "loss": 7.668904228344839, "lr": 8.656882499999999e-05, "optimizer_lrs": [0.0057712549999999994, 8.656882499999999e-05], "gradient_norm": 1.8712865114212036, "seconds": 0.32422938800004886, "nonpadding_tokens_per_second": 195229.06418338136, "ranks": [{"compute_seconds": 0.3215664020000304, "nonpadding_tokens": 10186, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3177577129999918, "nonpadding_tokens": 5620, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31857948499998656, "nonpadding_tokens": 11321, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3219526960000394, "nonpadding_tokens": 7885, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32086062700000184, "nonpadding_tokens": 6463, "peak_allocated_bytes": 14970580480, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3178745139999819, "nonpadding_tokens": 10938, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3200515279999081, "nonpadding_tokens": 3411, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.31848109400004887, "nonpadding_tokens": 7475, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00035049200005232706, "prefetch": {"data_wait_seconds": 0.0005765840002140976, "consumed_batches": 59, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 64, "tokens": 5851633, "loss_tokens": 80379, "loss": 7.705398020627278, "lr": 8.777451e-05, "optimizer_lrs": [0.0058516340000000005, 8.777451e-05], "gradient_norm": 1.5911873579025269, "seconds": 0.31644438199998604, "nonpadding_tokens_per_second": 254006.7214718432, "ranks": [{"compute_seconds": 0.3101190830000178, "nonpadding_tokens": 13279, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30904179999993175, "nonpadding_tokens": 6986, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3084932919999801, "nonpadding_tokens": 13371, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3131034879999106, "nonpadding_tokens": 5935, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31169770799999696, "nonpadding_tokens": 11495, "peak_allocated_bytes": 14970580480, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3139832759999308, "nonpadding_tokens": 13301, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31023155599996244, "nonpadding_tokens": 12949, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.311762806000047, "nonpadding_tokens": 3063, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0004041459999371, "prefetch": {"data_wait_seconds": 0.0005875690002312695, "consumed_batches": 60, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 65, "tokens": 5935776, "loss_tokens": 84143, "loss": 7.781014909142769, "lr": 8.903665499999999e-05, "optimizer_lrs": [0.005935777, 8.903665499999999e-05], "gradient_norm": 1.433578372001648, "seconds": 0.4262098980000246, "nonpadding_tokens_per_second": 197421.5061518706, "ranks": [{"compute_seconds": 0.4201415500000394, "nonpadding_tokens": 10415, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.42384275299991714, "nonpadding_tokens": 11621, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.419315959999949, "nonpadding_tokens": 7287, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4223234370000455, "nonpadding_tokens": 9682, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.42130785699998796, "nonpadding_tokens": 18397, "peak_allocated_bytes": 14970580480, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.41894531100001586, "nonpadding_tokens": 3394, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.42116848499995285, "nonpadding_tokens": 8223, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.42221257699998205, "nonpadding_tokens": 15124, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00034724300007837883, "prefetch": {"data_wait_seconds": 0.0005969030003143416, "consumed_batches": 61, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 66, "tokens": 6023214, "loss_tokens": 87438, "loss": 7.643297250623299, "lr": 9.034822499999999e-05, "optimizer_lrs": [0.006023215, 9.034822499999999e-05], "gradient_norm": 1.7344560623168945, "seconds": 0.34513136200007466, "nonpadding_tokens_per_second": 253347.01399863246, "ranks": [{"compute_seconds": 0.30155780600000526, "nonpadding_tokens": 15470, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31774908800002777, "nonpadding_tokens": 13490, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3166140800000221, "nonpadding_tokens": 15135, "peak_allocated_bytes": 14984101376, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32991970099999435, "nonpadding_tokens": 3138, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31676963699999305, "nonpadding_tokens": 6077, "peak_allocated_bytes": 14970580480, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.313901718000011, "nonpadding_tokens": 5112, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3171265900000435, "nonpadding_tokens": 17262, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3171076489999223, "nonpadding_tokens": 11754, "peak_allocated_bytes": 14978809344, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0004020330000003014, "prefetch": {"data_wait_seconds": 0.0006059250002863337, "consumed_batches": 62, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 67, "tokens": 6113459, "loss_tokens": 90245, "loss": 7.620219264225165, "lr": 9.170189999999999e-05, "optimizer_lrs": [0.006113459999999999, 9.170189999999999e-05], "gradient_norm": 1.3981287479400635, "seconds": 0.27808747500000663, "nonpadding_tokens_per_second": 324520.18919585587, "ranks": [{"compute_seconds": 0.21943674700003157, "nonpadding_tokens": 2486, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.231276960999935, "nonpadding_tokens": 25950, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.22785971099995095, "nonpadding_tokens": 12820, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.22794809999993504, "nonpadding_tokens": 10719, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.23500635799996417, "nonpadding_tokens": 4238, "peak_allocated_bytes": 14970580480, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.22735133100002258, "nonpadding_tokens": 10930, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2374713269999802, "nonpadding_tokens": 8240, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.22819196499995087, "nonpadding_tokens": 14862, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.02793305699992743, "prefetch": {"data_wait_seconds": 0.0006175910002639284, "consumed_batches": 63, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 68, "tokens": 6180202, "loss_tokens": 66743, "loss": 7.499347309830244, "lr": 9.270304499999999e-05, "optimizer_lrs": [0.0061802029999999996, 9.270304499999999e-05], "gradient_norm": 1.9267216920852661, "seconds": 0.34649143799993, "nonpadding_tokens_per_second": 192625.25038212774, "ranks": [{"compute_seconds": 0.34094959399999425, "nonpadding_tokens": 17012, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.342695195000033, "nonpadding_tokens": 5154, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3431695020000234, "nonpadding_tokens": 3498, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3440613760000133, "nonpadding_tokens": 5357, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34168168600001536, "nonpadding_tokens": 12426, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.33867923999991945, "nonpadding_tokens": 3038, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3414933139999903, "nonpadding_tokens": 3359, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3408702850000509, "nonpadding_tokens": 16899, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003925250000520464, "prefetch": {"data_wait_seconds": 0.0006293340002230252, "consumed_batches": 64, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 69, "tokens": 6311433, "loss_tokens": 131231, "loss": 7.212793185299205, "lr": 9.467151e-05, "optimizer_lrs": [0.006311434, 9.467151e-05], "gradient_norm": 1.93954598903656, "seconds": 0.31838964500002476, "nonpadding_tokens_per_second": 412171.0679378087, "ranks": [{"compute_seconds": 0.312839466000014, "nonpadding_tokens": 6346, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3108804999999393, "nonpadding_tokens": 21366, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31189856000003147, "nonpadding_tokens": 11406, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31058254300000954, "nonpadding_tokens": 27327, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31385954300003505, "nonpadding_tokens": 21661, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3159820600000103, "nonpadding_tokens": 12836, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3141813120000734, "nonpadding_tokens": 23791, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3141159269999889, "nonpadding_tokens": 6498, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003861179999375963, "prefetch": {"data_wait_seconds": 0.0006385040002214737, "consumed_batches": 65, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 70, "tokens": 6401708, "loss_tokens": 90275, "loss": 7.582875934644143, "lr": 9.602563499999999e-05, "optimizer_lrs": [0.006401708999999999, 9.602563499999999e-05], "gradient_norm": 1.6463559865951538, "seconds": 0.3709085680000044, "nonpadding_tokens_per_second": 243388.82352267185, "ranks": [{"compute_seconds": 0.36624264499994297, "nonpadding_tokens": 21447, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3686121570000296, "nonpadding_tokens": 7228, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36746123499995065, "nonpadding_tokens": 8341, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36460286700003053, "nonpadding_tokens": 9724, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36559528899999805, "nonpadding_tokens": 18985, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3653190219999942, "nonpadding_tokens": 7631, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.36766389299998536, "nonpadding_tokens": 10204, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.36766868900008376, "nonpadding_tokens": 6715, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00037796399999479036, "prefetch": {"data_wait_seconds": 0.0006472790001907924, "consumed_batches": 66, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 71, "tokens": 6451847, "loss_tokens": 50139, "loss": 7.406564126727697, "lr": 9.677772e-05, "optimizer_lrs": [0.0064518480000000005, 9.677772e-05], "gradient_norm": 2.5940494537353516, "seconds": 0.3427002990000574, "nonpadding_tokens_per_second": 146305.67917885477, "ranks": [{"compute_seconds": 0.3370624859999225, "nonpadding_tokens": 4370, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3395711340000389, "nonpadding_tokens": 5414, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3400716689999399, "nonpadding_tokens": 4131, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3399463640000704, "nonpadding_tokens": 2298, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3381547029999865, "nonpadding_tokens": 6943, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.340268229000003, "nonpadding_tokens": 4636, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33644671299998663, "nonpadding_tokens": 11336, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.33855150599993067, "nonpadding_tokens": 11011, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00034976100005224, "prefetch": {"data_wait_seconds": 0.0006561140002077082, "consumed_batches": 67, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 72, "tokens": 6521725, "loss_tokens": 69878, "loss": 7.476962080340021, "lr": 9.782588999999999e-05, "optimizer_lrs": [0.006521725999999999, 9.782588999999999e-05], "gradient_norm": 1.3093125820159912, "seconds": 0.28506789700008994, "nonpadding_tokens_per_second": 245127.5669247946, "ranks": [{"compute_seconds": 0.2790245830000231, "nonpadding_tokens": 7010, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2791408099999444, "nonpadding_tokens": 5641, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.27829468000004454, "nonpadding_tokens": 1842, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2821109889999889, "nonpadding_tokens": 14844, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2806468290000339, "nonpadding_tokens": 6838, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.2815257480000355, "nonpadding_tokens": 9843, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.28072892700004104, "nonpadding_tokens": 14661, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.2808064280000053, "nonpadding_tokens": 9199, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0003812360000665649, "prefetch": {"data_wait_seconds": 0.0006654440002193951, "consumed_batches": 68, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 73, "tokens": 6621472, "loss_tokens": 99747, "loss": 7.286051836145448, "lr": 9.9322095e-05, "optimizer_lrs": [0.006621473, 9.9322095e-05], "gradient_norm": 1.468713402748108, "seconds": 0.4006743940000206, "nonpadding_tokens_per_second": 248947.77778086532, "ranks": [{"compute_seconds": 0.39709012500009067, "nonpadding_tokens": 20154, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39681445300004725, "nonpadding_tokens": 16469, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.39347940799996195, "nonpadding_tokens": 13658, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3933144700000639, "nonpadding_tokens": 9526, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3942056279998951, "nonpadding_tokens": 19231, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3982072139999673, "nonpadding_tokens": 6315, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.39577450100000533, "nonpadding_tokens": 7896, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.395872347999898, "nonpadding_tokens": 6498, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00038773400001446134, "prefetch": {"data_wait_seconds": 0.0006751360001544526, "consumed_batches": 69, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 74, "tokens": 6713331, "loss_tokens": 91859, "loss": 7.2716296171306025, "lr": 0.00010069997999999998, "optimizer_lrs": [0.006713331999999999, 0.00010069997999999998], "gradient_norm": 1.4743657112121582, "seconds": 0.2693763140000556, "nonpadding_tokens_per_second": 341006.2252168951, "ranks": [{"compute_seconds": 0.2646549490000325, "nonpadding_tokens": 7914, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.266761353999982, "nonpadding_tokens": 13318, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.26319212800001424, "nonpadding_tokens": 12850, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.26698122699997384, "nonpadding_tokens": 8975, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2657779109999865, "nonpadding_tokens": 21709, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.26250282299997707, "nonpadding_tokens": 9801, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2637186819999897, "nonpadding_tokens": 9389, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.2629710190000196, "nonpadding_tokens": 7903, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.0004142000000229018, "prefetch": {"data_wait_seconds": 0.0006867430001875618, "consumed_batches": 70, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 75, "tokens": 6811354, "loss_tokens": 98023, "loss": 7.168340593534171, "lr": 0.000102170325, "optimizer_lrs": [0.006811355, 0.000102170325], "gradient_norm": 2.421638250350952, "seconds": 0.4108386289999544, "nonpadding_tokens_per_second": 238592.46205402678, "ranks": [{"compute_seconds": 0.40487863200007723, "nonpadding_tokens": 21183, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40846788399994693, "nonpadding_tokens": 10573, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.40749414999993405, "nonpadding_tokens": 2977, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.40433756599998105, "nonpadding_tokens": 12980, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.40454456999998456, "nonpadding_tokens": 19642, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.404343874999995, "nonpadding_tokens": 9153, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4077045269999644, "nonpadding_tokens": 13284, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.4071392050000213, "nonpadding_tokens": 8231, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21183332352}], "data_fetch_seconds": 0.00037644400003955525, "prefetch": {"data_wait_seconds": 0.0006960160001199256, "consumed_batches": 71, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 76, "tokens": 6880321, "loss_tokens": 68967, "loss": 7.119455681702844, "lr": 0.00010320482999999999, "optimizer_lrs": [0.0068803219999999995, 0.00010320482999999999], "gradient_norm": 1.8373396396636963, "seconds": 0.3175797850000208, "nonpadding_tokens_per_second": 217164.32612357705, "ranks": [{"compute_seconds": 0.3114182479999954, "nonpadding_tokens": 9069, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3149737720000303, "nonpadding_tokens": 4143, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31129944299993895, "nonpadding_tokens": 11204, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3152568609999662, "nonpadding_tokens": 7863, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3134190740000804, "nonpadding_tokens": 5410, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3112822099999448, "nonpadding_tokens": 15235, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31160881799996787, "nonpadding_tokens": 8869, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19620954112}, {"compute_seconds": 0.3115273299999899, "nonpadding_tokens": 7174, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00034393699991142057, "prefetch": {"data_wait_seconds": 0.0007049160001315613, "consumed_batches": 72, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 77, "tokens": 6982008, "loss_tokens": 101687, "loss": 7.285153215258588, "lr": 0.000104730135, "optimizer_lrs": [0.006982009, 0.000104730135], "gradient_norm": 1.3403823375701904, "seconds": 0.3754140489999145, "nonpadding_tokens_per_second": 270866.2615874111, "ranks": [{"compute_seconds": 0.37060834800001885, "nonpadding_tokens": 17472, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.37273891900008493, "nonpadding_tokens": 10889, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3682303350000211, "nonpadding_tokens": 14000, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36724204800009375, "nonpadding_tokens": 9728, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3699016999999003, "nonpadding_tokens": 16654, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.36797765599999366, "nonpadding_tokens": 16852, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3710885899999994, "nonpadding_tokens": 8267, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.37059827199993833, "nonpadding_tokens": 7825, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003487559999939549, "prefetch": {"data_wait_seconds": 0.0007145550000586809, "consumed_batches": 73, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 78, "tokens": 7038790, "loss_tokens": 56782, "loss": 7.204062246838787, "lr": 0.00010558186499999999, "optimizer_lrs": [0.007038791, 0.00010558186499999999], "gradient_norm": 1.3518129587173462, "seconds": 0.3829311840000855, "nonpadding_tokens_per_second": 148282.5175188326, "ranks": [{"compute_seconds": 0.37871130299993183, "nonpadding_tokens": 9205, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3799282960000028, "nonpadding_tokens": 2873, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.37658787700001994, "nonpadding_tokens": 10321, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38053631000002497, "nonpadding_tokens": 2584, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3786771119999912, "nonpadding_tokens": 9597, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3803476660000342, "nonpadding_tokens": 11416, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3794380550000369, "nonpadding_tokens": 7579, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.37894788700009485, "nonpadding_tokens": 3207, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00037131599992790143, "prefetch": {"data_wait_seconds": 0.0007235490000994105, "consumed_batches": 74, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 79, "tokens": 7106399, "loss_tokens": 67609, "loss": 7.014320855951131, "lr": 0.000106596, "optimizer_lrs": [0.0071064, 0.000106596], "gradient_norm": 1.520695447921753, "seconds": 0.3665332009999247, "nonpadding_tokens_per_second": 184455.32305275093, "ranks": [{"compute_seconds": 0.36275416599994514, "nonpadding_tokens": 3749, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.36417455499997686, "nonpadding_tokens": 8747, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36235588600004576, "nonpadding_tokens": 4167, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35974369999996725, "nonpadding_tokens": 6088, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36279672499995286, "nonpadding_tokens": 13385, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3605592249999745, "nonpadding_tokens": 12494, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3632632659999899, "nonpadding_tokens": 9343, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3609187109999539, "nonpadding_tokens": 9636, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0004017159999420983, "prefetch": {"data_wait_seconds": 0.0007345640001403808, "consumed_batches": 75, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 80, "tokens": 7229109, "loss_tokens": 122710, "loss": 6.991389251079782, "lr": 0.00010843664999999998, "optimizer_lrs": [0.0072291099999999995, 0.00010843664999999998], "gradient_norm": 1.5760643482208252, "seconds": 0.32328550600004746, "nonpadding_tokens_per_second": 379571.6099934959, "ranks": [{"compute_seconds": 0.29060176099994806, "nonpadding_tokens": 6223, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3203737100000126, "nonpadding_tokens": 22892, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31706972300003144, "nonpadding_tokens": 25780, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2914331069999889, "nonpadding_tokens": 11561, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2895860060000359, "nonpadding_tokens": 10045, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.292713305999996, "nonpadding_tokens": 19551, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2874119770000334, "nonpadding_tokens": 10989, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.29030209299992293, "nonpadding_tokens": 15669, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00035626199996841024, "prefetch": {"data_wait_seconds": 0.0007434230001308606, "consumed_batches": 76, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 81, "tokens": 7291900, "loss_tokens": 62791, "loss": 7.095005155993693, "lr": 0.00010937851499999998, "optimizer_lrs": [0.007291900999999999, 0.00010937851499999998], "gradient_norm": 1.3021605014801025, "seconds": 0.32675977899998543, "nonpadding_tokens_per_second": 192162.5733502617, "ranks": [{"compute_seconds": 0.3230989879999697, "nonpadding_tokens": 7722, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32415526600004796, "nonpadding_tokens": 4501, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32077331599998615, "nonpadding_tokens": 2819, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32427030799999557, "nonpadding_tokens": 5649, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32412262799994096, "nonpadding_tokens": 6825, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3205704470000228, "nonpadding_tokens": 9265, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3238035750000563, "nonpadding_tokens": 12299, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.323438742999997, "nonpadding_tokens": 13711, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003650730000117619, "prefetch": {"data_wait_seconds": 0.000753887000200848, "consumed_batches": 77, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 82, "tokens": 7438293, "loss_tokens": 146393, "loss": 7.13536764052926, "lr": 0.00011157440999999999, "optimizer_lrs": [0.007438294, 0.00011157440999999999], "gradient_norm": 1.1142377853393555, "seconds": 0.39076395399990815, "nonpadding_tokens_per_second": 374632.81477603846, "ranks": [{"compute_seconds": 0.38767956800006687, "nonpadding_tokens": 23500, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.38455915900010496, "nonpadding_tokens": 14792, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.38541093800006365, "nonpadding_tokens": 16767, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38593304300002274, "nonpadding_tokens": 25126, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3879671210000879, "nonpadding_tokens": 17419, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3847790019999593, "nonpadding_tokens": 26675, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.38661270099999, "nonpadding_tokens": 7978, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.388451721000024, "nonpadding_tokens": 14136, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003845940000246628, "prefetch": {"data_wait_seconds": 0.0007630140001992913, "consumed_batches": 78, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 83, "tokens": 7579465, "loss_tokens": 141172, "loss": 7.0491373112231885, "lr": 0.00011369199, "optimizer_lrs": [0.007579466, 0.00011369199], "gradient_norm": 1.1934369802474976, "seconds": 0.3918175210000072, "nonpadding_tokens_per_second": 360300.37564348074, "ranks": [{"compute_seconds": 0.38802450000002864, "nonpadding_tokens": 20954, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.38492584200002966, "nonpadding_tokens": 11180, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.38558978300000035, "nonpadding_tokens": 10703, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38940484099998685, "nonpadding_tokens": 18801, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.38811262799993074, "nonpadding_tokens": 16388, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.384474675999968, "nonpadding_tokens": 25762, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3849823279999782, "nonpadding_tokens": 17535, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.38808994699991217, "nonpadding_tokens": 19849, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0004004429999895365, "prefetch": {"data_wait_seconds": 0.0007739510001556482, "consumed_batches": 79, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 84, "tokens": 7676105, "loss_tokens": 96640, "loss": 7.111184550910596, "lr": 0.00011514158999999999, "optimizer_lrs": [0.007676105999999999, 0.00011514158999999999], "gradient_norm": 0.9864575862884521, "seconds": 0.3538814300000013, "nonpadding_tokens_per_second": 273085.8186031396, "ranks": [{"compute_seconds": 0.3493627620000552, "nonpadding_tokens": 19531, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3515192149999393, "nonpadding_tokens": 13785, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34817003400007707, "nonpadding_tokens": 7397, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34623104700006024, "nonpadding_tokens": 23513, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.349624674000097, "nonpadding_tokens": 10573, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3473301660000061, "nonpadding_tokens": 8059, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.35061137299999245, "nonpadding_tokens": 6433, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3478486139999859, "nonpadding_tokens": 7349, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00039979500002118584, "prefetch": {"data_wait_seconds": 0.0007842530001198611, "consumed_batches": 80, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 85, "tokens": 7761407, "loss_tokens": 85302, "loss": 6.497710340906426, "lr": 0.00011642111999999998, "optimizer_lrs": [0.007761407999999999, 0.00011642111999999998], "gradient_norm": 1.7077263593673706, "seconds": 0.301509966000026, "nonpadding_tokens_per_second": 282916.02142263064, "ranks": [{"compute_seconds": 0.2945743360000961, "nonpadding_tokens": 7620, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.29409153999995397, "nonpadding_tokens": 15466, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.29517210699998486, "nonpadding_tokens": 11268, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2991343290000259, "nonpadding_tokens": 15271, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.29572014600000784, "nonpadding_tokens": 15884, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.29854853400001957, "nonpadding_tokens": 5436, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.29752443600000333, "nonpadding_tokens": 4531, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.297801009000068, "nonpadding_tokens": 9826, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00036509200003820297, "prefetch": {"data_wait_seconds": 0.0007935280001447609, "consumed_batches": 81, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 86, "tokens": 7846899, "loss_tokens": 85492, "loss": 6.665981904739625, "lr": 0.00011770349999999999, "optimizer_lrs": [0.0078469, 0.00011770349999999999], "gradient_norm": 1.5980168581008911, "seconds": 0.3243778339999608, "nonpadding_tokens_per_second": 263556.8495719419, "ranks": [{"compute_seconds": 0.318606122999995, "nonpadding_tokens": 9974, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32201720299997305, "nonpadding_tokens": 9517, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3178619020000042, "nonpadding_tokens": 12911, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3220786770000359, "nonpadding_tokens": 17585, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3199019500000304, "nonpadding_tokens": 8151, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19230883840}, {"compute_seconds": 0.3179045000000542, "nonpadding_tokens": 7857, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32100045599997884, "nonpadding_tokens": 7277, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3186093740000615, "nonpadding_tokens": 12220, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003805959998999242, "prefetch": {"data_wait_seconds": 0.0008020910001960146, "consumed_batches": 82, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 87, "tokens": 7925630, "loss_tokens": 78731, "loss": 6.753307940963534, "lr": 0.00011888446499999998, "optimizer_lrs": [0.007925630999999999, 0.00011888446499999998], "gradient_norm": 1.1303296089172363, "seconds": 0.2997590530000025, "nonpadding_tokens_per_second": 262647.6138487112, "ranks": [{"compute_seconds": 0.2941383220000944, "nonpadding_tokens": 7304, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.29563930699998764, "nonpadding_tokens": 12740, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2927219990000367, "nonpadding_tokens": 2203, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.29568608399995355, "nonpadding_tokens": 18079, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.29547456199998123, "nonpadding_tokens": 9065, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.29737386499994045, "nonpadding_tokens": 13387, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.29561998400004086, "nonpadding_tokens": 4355, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.29248381900004006, "nonpadding_tokens": 11598, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003485789999331246, "prefetch": {"data_wait_seconds": 0.0008118200001945297, "consumed_batches": 83, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 88, "tokens": 8031687, "loss_tokens": 106057, "loss": 6.651663727995324, "lr": 0.00012047532, "optimizer_lrs": [0.008031688, 0.00012047532], "gradient_norm": 1.1389882564544678, "seconds": 0.44659468600002583, "nonpadding_tokens_per_second": 237479.31474490018, "ranks": [{"compute_seconds": 0.4362427220000882, "nonpadding_tokens": 9271, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4341469779999443, "nonpadding_tokens": 5579, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.43456700299998374, "nonpadding_tokens": 13539, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4385797549999779, "nonpadding_tokens": 22083, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4370401670000774, "nonpadding_tokens": 12882, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.43831765999993877, "nonpadding_tokens": 9905, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.43573613500007013, "nonpadding_tokens": 19691, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.4372178520000034, "nonpadding_tokens": 13107, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.006094634999953996, "prefetch": {"data_wait_seconds": 0.0008211880001454119, "consumed_batches": 84, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 89, "tokens": 8110215, "loss_tokens": 78528, "loss": 6.7298535235839445, "lr": 0.00012165323999999999, "optimizer_lrs": [0.008110216, 0.00012165323999999999], "gradient_norm": 1.894421935081482, "seconds": 0.302996564999944, "nonpadding_tokens_per_second": 259171.25496130466, "ranks": [{"compute_seconds": 0.2989921979999508, "nonpadding_tokens": 11750, "peak_allocated_bytes": 14998456832, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2988775540000006, "nonpadding_tokens": 2750, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2992229120000047, "nonpadding_tokens": 9711, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.295245465999983, "nonpadding_tokens": 9226, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.298181483999997, "nonpadding_tokens": 16550, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30060929499995837, "nonpadding_tokens": 9656, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.29860637800004497, "nonpadding_tokens": 4618, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.29857925800001794, "nonpadding_tokens": 14267, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003857829999560636, "prefetch": {"data_wait_seconds": 0.0008301040001015281, "consumed_batches": 85, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 90, "tokens": 8195572, "loss_tokens": 85357, "loss": 6.623744244760243, "lr": 0.000122933595, "optimizer_lrs": [0.008195573, 0.000122933595], "gradient_norm": 1.4233182668685913, "seconds": 0.3258595309999919, "nonpadding_tokens_per_second": 261944.1565451775, "ranks": [{"compute_seconds": 0.321021121000058, "nonpadding_tokens": 16676, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3186370700000225, "nonpadding_tokens": 9784, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.318423431000042, "nonpadding_tokens": 6668, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32255278799993903, "nonpadding_tokens": 12759, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32109376200003226, "nonpadding_tokens": 9386, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.323429109000017, "nonpadding_tokens": 10027, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31898093500001323, "nonpadding_tokens": 16600, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.32052063100002215, "nonpadding_tokens": 3457, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003590870001062285, "prefetch": {"data_wait_seconds": 0.000840822000100161, "consumed_batches": 86, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 91, "tokens": 8285111, "loss_tokens": 89539, "loss": 6.5303094182423305, "lr": 0.00012427667999999998, "optimizer_lrs": [0.008285111999999999, 0.00012427667999999998], "gradient_norm": 1.2892228364944458, "seconds": 0.2673653619999641, "nonpadding_tokens_per_second": 334893.79226323275, "ranks": [{"compute_seconds": 0.26292570600003273, "nonpadding_tokens": 13203, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2596421559999271, "nonpadding_tokens": 15248, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.26066282599992974, "nonpadding_tokens": 19364, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2650913679999576, "nonpadding_tokens": 13481, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2630249639998965, "nonpadding_tokens": 4956, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.26066120999996656, "nonpadding_tokens": 2997, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2634619100000464, "nonpadding_tokens": 9098, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.26160197000001517, "nonpadding_tokens": 11192, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00035739299994475004, "prefetch": {"data_wait_seconds": 0.000850746000082836, "consumed_batches": 87, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 92, "tokens": 8375522, "loss_tokens": 90411, "loss": 6.610979996903032, "lr": 0.000125632845, "optimizer_lrs": [0.008375523, 0.000125632845], "gradient_norm": 1.195096731185913, "seconds": 0.3349941190000436, "nonpadding_tokens_per_second": 269888.32003939815, "ranks": [{"compute_seconds": 0.32981392600004256, "nonpadding_tokens": 16719, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33162456900004145, "nonpadding_tokens": 9408, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3305391559999862, "nonpadding_tokens": 16960, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33166537700003573, "nonpadding_tokens": 10476, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3305431999999655, "nonpadding_tokens": 10638, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.332552731000078, "nonpadding_tokens": 10305, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33053793900000983, "nonpadding_tokens": 5052, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3304803940000056, "nonpadding_tokens": 10853, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00040321799997400376, "prefetch": {"data_wait_seconds": 0.0008611400000972935, "consumed_batches": 88, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 93, "tokens": 8425298, "loss_tokens": 49776, "loss": 6.6405433843619415, "lr": 0.000126379485, "optimizer_lrs": [0.008425299, 0.000126379485], "gradient_norm": 1.0518138408660889, "seconds": 0.30671950199996445, "nonpadding_tokens_per_second": 162285.08352235708, "ranks": [{"compute_seconds": 0.30177280899999914, "nonpadding_tokens": 7154, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30333476800001336, "nonpadding_tokens": 10313, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30078510600003483, "nonpadding_tokens": 6941, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3044043510000165, "nonpadding_tokens": 6034, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2998250519999601, "nonpadding_tokens": 5344, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.29978937100008807, "nonpadding_tokens": 3245, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.303366088999951, "nonpadding_tokens": 4001, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3026845960000628, "nonpadding_tokens": 6744, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003492399999913687, "prefetch": {"data_wait_seconds": 0.000870262000034927, "consumed_batches": 89, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 94, "tokens": 8502484, "loss_tokens": 77186, "loss": 6.666541293110149, "lr": 0.00012753727499999998, "optimizer_lrs": [0.008502484999999999, 0.00012753727499999998], "gradient_norm": 1.2043609619140625, "seconds": 0.45129704999999376, "nonpadding_tokens_per_second": 171031.47472380125, "ranks": [{"compute_seconds": 0.44624976199997946, "nonpadding_tokens": 12327, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4485626069999853, "nonpadding_tokens": 12262, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4448536510000167, "nonpadding_tokens": 5164, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.44883347200004664, "nonpadding_tokens": 6961, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4447873950000485, "nonpadding_tokens": 6144, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4436651510000047, "nonpadding_tokens": 6894, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4472353610000255, "nonpadding_tokens": 15178, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.44664762900004007, "nonpadding_tokens": 12256, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003449519999776385, "prefetch": {"data_wait_seconds": 0.0008794469999884313, "consumed_batches": 90, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 95, "tokens": 8547392, "loss_tokens": 44908, "loss": 6.556753529437962, "lr": 0.000128210895, "optimizer_lrs": [0.008547393, 0.000128210895], "gradient_norm": 1.0917810201644897, "seconds": 0.23434716000008393, "nonpadding_tokens_per_second": 191630.22927175186, "ranks": [{"compute_seconds": 0.2275577959999282, "nonpadding_tokens": 7144, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.23112555399995927, "nonpadding_tokens": 2396, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.23004692799997883, "nonpadding_tokens": 9103, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.23093835199995283, "nonpadding_tokens": 3446, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2298203379999677, "nonpadding_tokens": 5478, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.23191715399991608, "nonpadding_tokens": 5791, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.22817126699999335, "nonpadding_tokens": 4603, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.22971031700001276, "nonpadding_tokens": 6947, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0004014149999420624, "prefetch": {"data_wait_seconds": 0.0008884039999657034, "consumed_batches": 91, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 96, "tokens": 8619079, "loss_tokens": 71687, "loss": 6.8300101482835105, "lr": 0.00012928619999999998, "optimizer_lrs": [0.00861908, 0.00012928619999999998], "gradient_norm": 1.6143407821655273, "seconds": 0.4003084130000616, "nonpadding_tokens_per_second": 179079.4239440303, "ranks": [{"compute_seconds": 0.3957590559999744, "nonpadding_tokens": 18513, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39795585800004574, "nonpadding_tokens": 11154, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3941769169999816, "nonpadding_tokens": 9271, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3936104010000463, "nonpadding_tokens": 822, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39682794600003035, "nonpadding_tokens": 4140, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.39402678900000865, "nonpadding_tokens": 18869, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.39406811200001357, "nonpadding_tokens": 2386, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3942212780000318, "nonpadding_tokens": 6532, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003583200000321085, "prefetch": {"data_wait_seconds": 0.000898579000022437, "consumed_batches": 92, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 97, "tokens": 8714978, "loss_tokens": 95899, "loss": 6.582282531621811, "lr": 0.000130724685, "optimizer_lrs": [0.008714979000000001, 0.000130724685], "gradient_norm": 1.0897985696792603, "seconds": 0.30685825800003386, "nonpadding_tokens_per_second": 312518.88290387613, "ranks": [{"compute_seconds": 0.3011709840000094, "nonpadding_tokens": 11426, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3002720849999605, "nonpadding_tokens": 14720, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3031183420000616, "nonpadding_tokens": 11947, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30448760399997354, "nonpadding_tokens": 20185, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30218187999992097, "nonpadding_tokens": 8965, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3002850259999832, "nonpadding_tokens": 4040, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3011722960000043, "nonpadding_tokens": 15740, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.30238719800001945, "nonpadding_tokens": 8876, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003585550000479998, "prefetch": {"data_wait_seconds": 0.0009064550000630334, "consumed_batches": 93, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 98, "tokens": 8794132, "loss_tokens": 79154, "loss": 6.589838321499861, "lr": 0.00013191199499999997, "optimizer_lrs": [0.008794132999999999, 0.00013191199499999997], "gradient_norm": 1.4339286088943481, "seconds": 0.31624552899995706, "nonpadding_tokens_per_second": 250292.86659104278, "ranks": [{"compute_seconds": 0.31094495599995753, "nonpadding_tokens": 6438, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3138166460000775, "nonpadding_tokens": 9545, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31280733899995994, "nonpadding_tokens": 17126, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3137347929999805, "nonpadding_tokens": 4300, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3123321970000461, "nonpadding_tokens": 4252, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30932214400002067, "nonpadding_tokens": 9230, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3126365209999449, "nonpadding_tokens": 9980, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3112405020000324, "nonpadding_tokens": 18283, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00036608099992463394, "prefetch": {"data_wait_seconds": 0.0009149880000904886, "consumed_batches": 94, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 99, "tokens": 8942633, "loss_tokens": 148501, "loss": 6.5500013467922775, "lr": 0.00013413951, "optimizer_lrs": [0.008942634, 0.00013413951], "gradient_norm": 4.0764336585998535, "seconds": 0.3737066189999041, "nonpadding_tokens_per_second": 397373.2132371948, "ranks": [{"compute_seconds": 0.3684597249999797, "nonpadding_tokens": 18152, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3713443599999664, "nonpadding_tokens": 30465, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36977529800003595, "nonpadding_tokens": 25908, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3667528290000064, "nonpadding_tokens": 12336, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36985977799997727, "nonpadding_tokens": 18436, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3671439030000556, "nonpadding_tokens": 14743, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.37067640299994764, "nonpadding_tokens": 15105, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3703194900000426, "nonpadding_tokens": 13356, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0004327770000145392, "prefetch": {"data_wait_seconds": 0.000939095000035195, "consumed_batches": 95, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 100, "tokens": 9051289, "loss_tokens": 108656, "loss": 6.438198879951407, "lr": 0.00013576934999999998, "optimizer_lrs": [0.009051289999999998, 0.00013576934999999998], "gradient_norm": 1.1553783416748047, "seconds": 0.3490012720000095, "nonpadding_tokens_per_second": 311334.1088338413, "ranks": [{"compute_seconds": 0.3446825340000714, "nonpadding_tokens": 16960, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34255196699996304, "nonpadding_tokens": 5460, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3438211179999371, "nonpadding_tokens": 13694, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3427275279999549, "nonpadding_tokens": 29327, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3457708549999552, "nonpadding_tokens": 13291, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3432795260000603, "nonpadding_tokens": 10832, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3442633739999792, "nonpadding_tokens": 6993, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.34676245800005745, "nonpadding_tokens": 12099, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00038762600001973624, "prefetch": {"data_wait_seconds": 0.0009480820000362655, "consumed_batches": 96, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 101, "tokens": 9121230, "loss_tokens": 69941, "loss": 6.741924443459487, "lr": 0.000136818465, "optimizer_lrs": [0.009121231, 0.000136818465], "gradient_norm": 0.9043883681297302, "seconds": 0.3141183679999813, "nonpadding_tokens_per_second": 222658.10320268877, "ranks": [{"compute_seconds": 0.30929270499996164, "nonpadding_tokens": 14233, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30722224600003756, "nonpadding_tokens": 3072, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30798975899995185, "nonpadding_tokens": 8569, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3117853679999598, "nonpadding_tokens": 10791, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31038957099997333, "nonpadding_tokens": 9338, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3076288319999776, "nonpadding_tokens": 10133, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31077682000000095, "nonpadding_tokens": 1459, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.30833692600003815, "nonpadding_tokens": 12346, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00034725800003343466, "prefetch": {"data_wait_seconds": 0.0009577239999316589, "consumed_batches": 97, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 102, "tokens": 9223386, "loss_tokens": 102156, "loss": 6.665537878342927, "lr": 0.00013835080499999998, "optimizer_lrs": [0.009223387, 0.00013835080499999998], "gradient_norm": 1.709625482559204, "seconds": 0.33482625800002097, "nonpadding_tokens_per_second": 305101.5192482114, "ranks": [{"compute_seconds": 0.33041749400001663, "nonpadding_tokens": 14380, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3282582059999868, "nonpadding_tokens": 6985, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32835582499990323, "nonpadding_tokens": 12466, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3324920340000972, "nonpadding_tokens": 19585, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3315192580000712, "nonpadding_tokens": 10200, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3287672479999628, "nonpadding_tokens": 9555, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32989607700005763, "nonpadding_tokens": 6929, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.33165826699996614, "nonpadding_tokens": 22056, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003681309999592486, "prefetch": {"data_wait_seconds": 0.0009678729999222924, "consumed_batches": 98, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 103, "tokens": 9313549, "loss_tokens": 90163, "loss": 6.435212476292936, "lr": 0.00013970325, "optimizer_lrs": [0.00931355, 0.00013970325], "gradient_norm": 1.3788849115371704, "seconds": 0.2704876240001113, "nonpadding_tokens_per_second": 333335.0290361636, "ranks": [{"compute_seconds": 0.26804371100001845, "nonpadding_tokens": 7690, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.26355674999990697, "nonpadding_tokens": 5390, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.264641719999986, "nonpadding_tokens": 13776, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2673016360000702, "nonpadding_tokens": 18091, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2640133419999984, "nonpadding_tokens": 7710, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.26349840900002164, "nonpadding_tokens": 10401, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2672860689999652, "nonpadding_tokens": 12430, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.26611625099997127, "nonpadding_tokens": 14675, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003485040000441586, "prefetch": {"data_wait_seconds": 0.0009775029999445906, "consumed_batches": 99, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 104, "tokens": 9384288, "loss_tokens": 70739, "loss": 6.3633974893623035, "lr": 0.000140764335, "optimizer_lrs": [0.009384289, 0.000140764335], "gradient_norm": 1.3621138334274292, "seconds": 0.2688894100000425, "nonpadding_tokens_per_second": 263078.41576947493, "ranks": [{"compute_seconds": 0.26518897599999036, "nonpadding_tokens": 4652, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.26651158799995756, "nonpadding_tokens": 2559, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2651909410000144, "nonpadding_tokens": 9634, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.26654549199997746, "nonpadding_tokens": 9959, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.265375449999965, "nonpadding_tokens": 6728, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.26245659699998214, "nonpadding_tokens": 16610, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.26550495100002536, "nonpadding_tokens": 12690, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.26522692999992614, "nonpadding_tokens": 7907, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00037685600000259, "prefetch": {"data_wait_seconds": 0.0009884779999538296, "consumed_batches": 100, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 105, "tokens": 9446852, "loss_tokens": 62564, "loss": 6.501310657886324, "lr": 0.00014170279499999997, "optimizer_lrs": [0.009446852999999998, 0.00014170279499999997], "gradient_norm": 1.5470569133758545, "seconds": 0.2808610959999669, "nonpadding_tokens_per_second": 222757.8005321441, "ranks": [{"compute_seconds": 0.2756033440000465, "nonpadding_tokens": 10987, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.27391289599995616, "nonpadding_tokens": 2569, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.27587682199998653, "nonpadding_tokens": 9668, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.27849058700007845, "nonpadding_tokens": 3366, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.27411211399999047, "nonpadding_tokens": 9286, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2743709609999314, "nonpadding_tokens": 4595, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.27751685199996246, "nonpadding_tokens": 10250, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.27659663499991893, "nonpadding_tokens": 11843, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00038484900005641975, "prefetch": {"data_wait_seconds": 0.0009988759999259855, "consumed_batches": 101, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 106, "tokens": 9518043, "loss_tokens": 71191, "loss": 6.29380469441362, "lr": 0.00014277065999999998, "optimizer_lrs": [0.009518044, 0.00014277065999999998], "gradient_norm": 1.0571070909500122, "seconds": 0.27841827400004604, "nonpadding_tokens_per_second": 255698.01499447637, "ranks": [{"compute_seconds": 0.27399984300006963, "nonpadding_tokens": 5570, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.27483652200010056, "nonpadding_tokens": 19215, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.271141236999938, "nonpadding_tokens": 7174, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.27448262900009013, "nonpadding_tokens": 3341, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.27417666899998494, "nonpadding_tokens": 3185, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.27600456599998324, "nonpadding_tokens": 9274, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2744820249999975, "nonpadding_tokens": 11556, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.27179535099992336, "nonpadding_tokens": 11876, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00040025900000273396, "prefetch": {"data_wait_seconds": 0.0010088529999165985, "consumed_batches": 102, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 107, "tokens": 9575960, "loss_tokens": 57917, "loss": 6.549695037726401, "lr": 0.00014363941499999998, "optimizer_lrs": [0.009575961, 0.00014363941499999998], "gradient_norm": 1.0002919435501099, "seconds": 0.3339622059999101, "nonpadding_tokens_per_second": 173423.81550808053, "ranks": [{"compute_seconds": 0.3290269650000255, "nonpadding_tokens": 16801, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33037607900007515, "nonpadding_tokens": 6554, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32680142700007764, "nonpadding_tokens": 11991, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.330419605999964, "nonpadding_tokens": 7934, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3266701509999166, "nonpadding_tokens": 3364, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3315479420000429, "nonpadding_tokens": 1870, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.329415154000003, "nonpadding_tokens": 3701, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3275240440000289, "nonpadding_tokens": 5702, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00034307700002500496, "prefetch": {"data_wait_seconds": 0.0010183399998595632, "consumed_batches": 103, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 108, "tokens": 9632266, "loss_tokens": 56306, "loss": 6.532958299293148, "lr": 0.000144484005, "optimizer_lrs": [0.009632267, 0.000144484005], "gradient_norm": 1.369545817375183, "seconds": 0.339267781999979, "nonpadding_tokens_per_second": 165963.29798272296, "ranks": [{"compute_seconds": 0.33535162400005447, "nonpadding_tokens": 7580, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33534827300002235, "nonpadding_tokens": 1586, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33536441499995817, "nonpadding_tokens": 4388, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3358226210000339, "nonpadding_tokens": 13106, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33272727499991106, "nonpadding_tokens": 11286, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3368412360000548, "nonpadding_tokens": 3591, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3346099629999344, "nonpadding_tokens": 6711, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.33394139899996844, "nonpadding_tokens": 8058, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00034325299998272385, "prefetch": {"data_wait_seconds": 0.0010281649998660214, "consumed_batches": 104, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 109, "tokens": 9691184, "loss_tokens": 58918, "loss": 6.772366360874436, "lr": 0.000145367775, "optimizer_lrs": [0.009691185000000001, 0.000145367775], "gradient_norm": 1.4975395202636719, "seconds": 0.31566890599992803, "nonpadding_tokens_per_second": 186644.92726443394, "ranks": [{"compute_seconds": 0.3108963989999438, "nonpadding_tokens": 1351, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31337699200003044, "nonpadding_tokens": 15295, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30915571599996383, "nonpadding_tokens": 16603, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30949173999999857, "nonpadding_tokens": 3213, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31243230499990204, "nonpadding_tokens": 3743, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30978124599994317, "nonpadding_tokens": 2000, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3106819989999394, "nonpadding_tokens": 9272, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3124952819999862, "nonpadding_tokens": 7441, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.00040156199997909425, "prefetch": {"data_wait_seconds": 0.0010366609998300191, "consumed_batches": 105, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 110, "tokens": 9751550, "loss_tokens": 60366, "loss": 6.580062659444058, "lr": 0.00014627326499999999, "optimizer_lrs": [0.009751550999999999, 0.00014627326499999999], "gradient_norm": 1.0656954050064087, "seconds": 0.3324799910000138, "nonpadding_tokens_per_second": 181562.80568474118, "ranks": [{"compute_seconds": 0.3286041850000174, "nonpadding_tokens": 13759, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3300241450000385, "nonpadding_tokens": 3590, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3262353650000023, "nonpadding_tokens": 6139, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3300965680000445, "nonpadding_tokens": 8528, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3288030070000332, "nonpadding_tokens": 11039, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3256517030000623, "nonpadding_tokens": 4916, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3290726610000547, "nonpadding_tokens": 5961, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3290144979999923, "nonpadding_tokens": 6434, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003640810000433703, "prefetch": {"data_wait_seconds": 0.0010467609998840999, "consumed_batches": 106, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 111, "tokens": 9825342, "loss_tokens": 73792, "loss": 6.447670479184736, "lr": 0.00014738014499999998, "optimizer_lrs": [0.009825342999999999, 0.00014738014499999998], "gradient_norm": 1.1092628240585327, "seconds": 0.30739355699995485, "nonpadding_tokens_per_second": 240057.0809622104, "ranks": [{"compute_seconds": 0.30085462900001403, "nonpadding_tokens": 3361, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30440242400004536, "nonpadding_tokens": 12225, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3006665720000683, "nonpadding_tokens": 3471, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30021777199999633, "nonpadding_tokens": 7543, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30306836700003714, "nonpadding_tokens": 12182, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3049322910000001, "nonpadding_tokens": 8036, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3037184999999454, "nonpadding_tokens": 16973, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.30372844199996507, "nonpadding_tokens": 10001, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003893389999802821, "prefetch": {"data_wait_seconds": 0.0010566059999064237, "consumed_batches": 107, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 112, "tokens": 9930365, "loss_tokens": 105023, "loss": 6.675223522466507, "lr": 0.00014895549, "optimizer_lrs": [0.009930366000000001, 0.00014895549], "gradient_norm": 1.6456122398376465, "seconds": 0.34533171299995047, "nonpadding_tokens_per_second": 304122.0833373478, "ranks": [{"compute_seconds": 0.3409091949999947, "nonpadding_tokens": 9612, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33917732999998407, "nonpadding_tokens": 11645, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3398898290000716, "nonpadding_tokens": 18697, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3392689000000928, "nonpadding_tokens": 26228, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3412409430000025, "nonpadding_tokens": 11845, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33964850500001376, "nonpadding_tokens": 11344, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.34253336300002957, "nonpadding_tokens": 9597, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.34231513800000357, "nonpadding_tokens": 6055, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003518780000604238, "prefetch": {"data_wait_seconds": 0.0010660929999630753, "consumed_batches": 108, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 113, "tokens": 10033217, "loss_tokens": 102852, "loss": 6.598277136079026, "lr": 0.00015049826999999997, "optimizer_lrs": [0.010033217999999998, 0.00015049826999999997], "gradient_norm": 0.9722124338150024, "seconds": 0.25733766099995137, "nonpadding_tokens_per_second": 399677.21630927327, "ranks": [{"compute_seconds": 0.2515274180000233, "nonpadding_tokens": 6854, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2511139319999529, "nonpadding_tokens": 12820, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2535215210000388, "nonpadding_tokens": 9757, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2548406039999236, "nonpadding_tokens": 7787, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2543722579999894, "nonpadding_tokens": 10733, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2516901470000903, "nonpadding_tokens": 7203, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2520561110000017, "nonpadding_tokens": 24835, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.2547812459999932, "nonpadding_tokens": 22863, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21256732672}], "data_fetch_seconds": 0.0003636719999349225, "prefetch": {"data_wait_seconds": 0.001076989000011963, "consumed_batches": 109, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 114, "tokens": 10119529, "loss_tokens": 86312, "loss": 6.5628700238669015, "lr": 0.00015179295, "optimizer_lrs": [0.010119530000000002, 0.00015179295], "gradient_norm": 1.1215003728866577, "seconds": 0.2975303510000913, "nonpadding_tokens_per_second": 290094.77423018776, "ranks": [{"compute_seconds": 0.24914783700000953, "nonpadding_tokens": 12744, "peak_allocated_bytes": 15004063232, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.266964061000067, "nonpadding_tokens": 8182, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2652490279999711, "nonpadding_tokens": 3301, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2675008069999194, "nonpadding_tokens": 15609, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.26553104199990685, "nonpadding_tokens": 17396, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.26280680599995776, "nonpadding_tokens": 4833, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.26355044699994323, "nonpadding_tokens": 13842, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.2637390250000635, "nonpadding_tokens": 10405, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.015470391999997446, "prefetch": {"data_wait_seconds": 0.0010857319999786341, "consumed_batches": 110, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 115, "tokens": 10203256, "loss_tokens": 83727, "loss": 6.387991627551447, "lr": 0.00015304885499999997, "optimizer_lrs": [0.010203256999999999, 0.00015304885499999997], "gradient_norm": 1.2229316234588623, "seconds": 0.29512472599992634, "nonpadding_tokens_per_second": 283700.39045803604, "ranks": [{"compute_seconds": 0.2905166430000463, "nonpadding_tokens": 10997, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2928439459999481, "nonpadding_tokens": 10749, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.29191067600004317, "nonpadding_tokens": 10099, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2927968599999531, "nonpadding_tokens": 15174, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2916669170000432, "nonpadding_tokens": 3069, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2879347059999873, "nonpadding_tokens": 17008, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.29196957099998144, "nonpadding_tokens": 5219, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.2921125139999958, "nonpadding_tokens": 11412, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036428099997465324, "prefetch": {"data_wait_seconds": 0.0010939839999082324, "consumed_batches": 111, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 116, "tokens": 10301767, "loss_tokens": 98511, "loss": 6.319706048055547, "lr": 0.00015452652, "optimizer_lrs": [0.010301768000000001, 0.00015452652], "gradient_norm": 1.2384626865386963, "seconds": 0.304806769000038, "nonpadding_tokens_per_second": 323191.6414559275, "ranks": [{"compute_seconds": 0.3013375900000028, "nonpadding_tokens": 14834, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3024230080000052, "nonpadding_tokens": 18533, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2981859509999367, "nonpadding_tokens": 11512, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.29734938199999306, "nonpadding_tokens": 7413, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3012214679999943, "nonpadding_tokens": 18254, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30169538000006924, "nonpadding_tokens": 3704, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3011510090000229, "nonpadding_tokens": 8865, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.29816055799994956, "nonpadding_tokens": 15396, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003423599999905491, "prefetch": {"data_wait_seconds": 0.0011032719999093388, "consumed_batches": 112, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 117, "tokens": 10372907, "loss_tokens": 71140, "loss": 6.617668945037953, "lr": 0.00015559362, "optimizer_lrs": [0.010372908, 0.00015559362], "gradient_norm": 0.9429057836532593, "seconds": 0.31535769900006017, "nonpadding_tokens_per_second": 225585.10613684566, "ranks": [{"compute_seconds": 0.309500387000071, "nonpadding_tokens": 9029, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3129763680000224, "nonpadding_tokens": 5166, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31182066799999575, "nonpadding_tokens": 4984, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3130054799999016, "nonpadding_tokens": 11338, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3102827769999976, "nonpadding_tokens": 9445, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3091745080000692, "nonpadding_tokens": 20856, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31064700100000664, "nonpadding_tokens": 7462, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3128379390000191, "nonpadding_tokens": 2860, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003879959999721905, "prefetch": {"data_wait_seconds": 0.0011110939999525726, "consumed_batches": 113, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 118, "tokens": 10440680, "loss_tokens": 67773, "loss": 6.340563812285128, "lr": 0.000156610215, "optimizer_lrs": [0.010440681, 0.000156610215], "gradient_norm": 0.9596172571182251, "seconds": 0.40449024199995165, "nonpadding_tokens_per_second": 167551.6315669442, "ranks": [{"compute_seconds": 0.39826564700001654, "nonpadding_tokens": 12119, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39751532299999326, "nonpadding_tokens": 11105, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3979290749999791, "nonpadding_tokens": 4993, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.40203021599995736, "nonpadding_tokens": 8843, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.40010562999998456, "nonpadding_tokens": 2229, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4017474329999686, "nonpadding_tokens": 10515, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4002409089999901, "nonpadding_tokens": 6848, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3982404159999078, "nonpadding_tokens": 11121, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034141600008297246, "prefetch": {"data_wait_seconds": 0.0011201699999219272, "consumed_batches": 114, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 119, "tokens": 10512785, "loss_tokens": 72105, "loss": 6.425027737327508, "lr": 0.00015769179, "optimizer_lrs": [0.010512786, 0.00015769179], "gradient_norm": 1.1137925386428833, "seconds": 0.31528259699996397, "nonpadding_tokens_per_second": 228699.58788118025, "ranks": [{"compute_seconds": 0.30852051500005473, "nonpadding_tokens": 7965, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3129612769999994, "nonpadding_tokens": 12982, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3095177689999673, "nonpadding_tokens": 18036, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3100653330000114, "nonpadding_tokens": 12406, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3086143819999734, "nonpadding_tokens": 3495, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3054006490000347, "nonpadding_tokens": 3208, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.30910432799998944, "nonpadding_tokens": 6557, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3070356000000629, "nonpadding_tokens": 7456, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003742820000525171, "prefetch": {"data_wait_seconds": 0.0011303459999680854, "consumed_batches": 115, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 120, "tokens": 10582645, "loss_tokens": 69860, "loss": 6.242431738476954, "lr": 0.00015873968999999998, "optimizer_lrs": [0.010582646, 0.00015873968999999998], "gradient_norm": 0.9569134712219238, "seconds": 0.35253388000000996, "nonpadding_tokens_per_second": 198165.35080258962, "ranks": [{"compute_seconds": 0.345769439000037, "nonpadding_tokens": 9483, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34904391700001725, "nonpadding_tokens": 14735, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34838034400002016, "nonpadding_tokens": 5348, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3493141089999199, "nonpadding_tokens": 1946, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34794809499999246, "nonpadding_tokens": 6226, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35013371600007304, "nonpadding_tokens": 21608, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.348494100000039, "nonpadding_tokens": 4600, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.34610426999995525, "nonpadding_tokens": 5914, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003773040000396577, "prefetch": {"data_wait_seconds": 0.0011400050000247575, "consumed_batches": 116, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 121, "tokens": 10670947, "loss_tokens": 88302, "loss": 6.605028057122149, "lr": 0.00016006421999999998, "optimizer_lrs": [0.010670948, 0.00016006421999999998], "gradient_norm": 0.8613656759262085, "seconds": 0.26816655200002515, "nonpadding_tokens_per_second": 329280.43912050495, "ranks": [{"compute_seconds": 0.2653208839999479, "nonpadding_tokens": 2305, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.26546036799993544, "nonpadding_tokens": 5049, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.26416720200006694, "nonpadding_tokens": 14968, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.26588040699994053, "nonpadding_tokens": 10605, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.263510745000076, "nonpadding_tokens": 10906, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2620350370000324, "nonpadding_tokens": 12819, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2648976829999583, "nonpadding_tokens": 25032, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.26480890600009843, "nonpadding_tokens": 6618, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003767350000316583, "prefetch": {"data_wait_seconds": 0.0011488100000178747, "consumed_batches": 117, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 122, "tokens": 10795125, "loss_tokens": 124178, "loss": 6.526363868801237, "lr": 0.00016192688999999998, "optimizer_lrs": [0.010795125999999999, 0.00016192688999999998], "gradient_norm": 0.8695826530456543, "seconds": 0.3821383810000043, "nonpadding_tokens_per_second": 324955.5819937349, "ranks": [{"compute_seconds": 0.37923863399998936, "nonpadding_tokens": 14396, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.37980238199997984, "nonpadding_tokens": 3970, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3761623760000248, "nonpadding_tokens": 10483, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37982318199999554, "nonpadding_tokens": 22692, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3784830890000421, "nonpadding_tokens": 9587, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37578035399997134, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.37651774300002216, "nonpadding_tokens": 15581, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.37842868399991403, "nonpadding_tokens": 14701, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003642230000195923, "prefetch": {"data_wait_seconds": 0.00115708300006645, "consumed_batches": 118, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 123, "tokens": 10873690, "loss_tokens": 78565, "loss": 6.336392636670273, "lr": 0.000163105365, "optimizer_lrs": [0.010873691000000001, 0.000163105365], "gradient_norm": 0.8984344005584717, "seconds": 0.2854286010000351, "nonpadding_tokens_per_second": 275252.7242355448, "ranks": [{"compute_seconds": 0.2796874250000201, "nonpadding_tokens": 9068, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2830532460000086, "nonpadding_tokens": 5348, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.27954210600000806, "nonpadding_tokens": 8004, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.27834440000003724, "nonpadding_tokens": 19061, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.27908571099999335, "nonpadding_tokens": 7574, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.27918231000001015, "nonpadding_tokens": 13047, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2801579079999783, "nonpadding_tokens": 14016, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.28203156200004287, "nonpadding_tokens": 2447, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00041500099996483186, "prefetch": {"data_wait_seconds": 0.0011680740001338563, "consumed_batches": 119, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 124, "tokens": 10957886, "loss_tokens": 84196, "loss": 6.181154686683453, "lr": 0.00016436830499999997, "optimizer_lrs": [0.010957887, 0.00016436830499999997], "gradient_norm": 1.1150622367858887, "seconds": 0.4220339470000454, "nonpadding_tokens_per_second": 199500.53923029787, "ranks": [{"compute_seconds": 0.41651769500003866, "nonpadding_tokens": 10565, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4196812589999581, "nonpadding_tokens": 2566, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4162984100000813, "nonpadding_tokens": 12472, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.41386333599996306, "nonpadding_tokens": 7006, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.41843451099998674, "nonpadding_tokens": 4769, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4153212029999622, "nonpadding_tokens": 17095, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.41899019700008466, "nonpadding_tokens": 11422, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.41837845599991397, "nonpadding_tokens": 18301, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035407699999723263, "prefetch": {"data_wait_seconds": 0.0011770770001930941, "consumed_batches": 120, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 125, "tokens": 11044647, "loss_tokens": 86761, "loss": 6.031070988116781, "lr": 0.00016566971999999998, "optimizer_lrs": [0.011044647999999999, 0.00016566971999999998], "gradient_norm": 1.1339671611785889, "seconds": 0.3409927500000549, "nonpadding_tokens_per_second": 254436.4946175132, "ranks": [{"compute_seconds": 0.33787697600007505, "nonpadding_tokens": 11223, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3382016640000529, "nonpadding_tokens": 11920, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3377476519999618, "nonpadding_tokens": 10117, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3386693940000214, "nonpadding_tokens": 4694, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3362790229999746, "nonpadding_tokens": 12735, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33423903100003827, "nonpadding_tokens": 14295, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33592521899993244, "nonpadding_tokens": 11882, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.33738475200004814, "nonpadding_tokens": 9895, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034279700003025937, "prefetch": {"data_wait_seconds": 0.0011862080001492359, "consumed_batches": 121, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 126, "tokens": 11145466, "loss_tokens": 100819, "loss": 6.242523730646009, "lr": 0.000167182005, "optimizer_lrs": [0.011145467000000001, 0.000167182005], "gradient_norm": 1.4660570621490479, "seconds": 0.3266084699999965, "nonpadding_tokens_per_second": 308684.584940498, "ranks": [{"compute_seconds": 0.32424425900001097, "nonpadding_tokens": 13781, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32410740600005283, "nonpadding_tokens": 20901, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32249680699999317, "nonpadding_tokens": 8606, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3204554479999615, "nonpadding_tokens": 6585, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3211855029999242, "nonpadding_tokens": 8623, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.320005110000011, "nonpadding_tokens": 10064, "peak_allocated_bytes": 14988325376, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3232878850000134, "nonpadding_tokens": 17019, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3229361370000561, "nonpadding_tokens": 15240, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004051949999848148, "prefetch": {"data_wait_seconds": 0.0011960250002402972, "consumed_batches": 122, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 127, "tokens": 11238579, "loss_tokens": 93113, "loss": 6.332965053214911, "lr": 0.00016857869999999998, "optimizer_lrs": [0.01123858, 0.00016857869999999998], "gradient_norm": 1.0849062204360962, "seconds": 0.3084263709999959, "nonpadding_tokens_per_second": 301897.0125612288, "ranks": [{"compute_seconds": 0.30509901499999614, "nonpadding_tokens": 10441, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30616365000003043, "nonpadding_tokens": 23685, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30171948700001394, "nonpadding_tokens": 8827, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3025268630000255, "nonpadding_tokens": 6002, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3039834390000351, "nonpadding_tokens": 7059, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30161095999994814, "nonpadding_tokens": 15378, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.30542371300009563, "nonpadding_tokens": 5548, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3051984539999921, "nonpadding_tokens": 16173, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003688709999778439, "prefetch": {"data_wait_seconds": 0.0012063630001648562, "consumed_batches": 123, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 128, "tokens": 11353963, "loss_tokens": 115384, "loss": 6.673713968141163, "lr": 0.00017030946, "optimizer_lrs": [0.011353964000000001, 0.00017030946], "gradient_norm": 1.7907648086547852, "seconds": 0.3483528339999111, "nonpadding_tokens_per_second": 331227.38998595154, "ranks": [{"compute_seconds": 0.33943474600005175, "nonpadding_tokens": 18347, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34243756299997585, "nonpadding_tokens": 19190, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33835379300001023, "nonpadding_tokens": 21080, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.342190458999994, "nonpadding_tokens": 9862, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3406495279999717, "nonpadding_tokens": 8665, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3422831049999786, "nonpadding_tokens": 14673, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3378017830000317, "nonpadding_tokens": 12594, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.34597390900000846, "nonpadding_tokens": 10973, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034873599997808924, "prefetch": {"data_wait_seconds": 0.0012145440001631869, "consumed_batches": 124, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 129, "tokens": 11441325, "loss_tokens": 87362, "loss": 6.316766729241547, "lr": 0.00017161989, "optimizer_lrs": [0.011441326, 0.00017161989], "gradient_norm": 1.236434817314148, "seconds": 0.3594119550000414, "nonpadding_tokens_per_second": 243069.2657398943, "ranks": [{"compute_seconds": 0.3524950700000318, "nonpadding_tokens": 12739, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35172346100000595, "nonpadding_tokens": 10511, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3544199320000416, "nonpadding_tokens": 14994, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3554227849999734, "nonpadding_tokens": 6460, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35260751400005574, "nonpadding_tokens": 8963, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35660978199996407, "nonpadding_tokens": 5235, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.354581641999971, "nonpadding_tokens": 19874, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.354148015000078, "nonpadding_tokens": 8586, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000513400999921032, "prefetch": {"data_wait_seconds": 0.0012254500001063207, "consumed_batches": 125, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 130, "tokens": 11521312, "loss_tokens": 79987, "loss": 5.964071896058109, "lr": 0.000172819695, "optimizer_lrs": [0.011521313, 0.000172819695], "gradient_norm": 1.2177661657333374, "seconds": 0.33908131999999114, "nonpadding_tokens_per_second": 235893.26595756467, "ranks": [{"compute_seconds": 0.33321228099998734, "nonpadding_tokens": 6757, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33126557699995374, "nonpadding_tokens": 6944, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33437647599998854, "nonpadding_tokens": 9702, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3318779120000954, "nonpadding_tokens": 9459, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33388518999993266, "nonpadding_tokens": 12868, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33124079900005654, "nonpadding_tokens": 14262, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33503204099997674, "nonpadding_tokens": 9469, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3368424100000311, "nonpadding_tokens": 10526, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035583600003974425, "prefetch": {"data_wait_seconds": 0.0012349060001497492, "consumed_batches": 126, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 131, "tokens": 11597525, "loss_tokens": 76213, "loss": 5.955767880807736, "lr": 0.00017396289, "optimizer_lrs": [0.011597526, 0.00017396289], "gradient_norm": 1.2367167472839355, "seconds": 0.36575231599999825, "nonpadding_tokens_per_second": 208373.25333573655, "ranks": [{"compute_seconds": 0.35734378099994046, "nonpadding_tokens": 3085, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3571858569999904, "nonpadding_tokens": 13160, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35770458100000724, "nonpadding_tokens": 11314, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3614086270000598, "nonpadding_tokens": 4686, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35876601200004643, "nonpadding_tokens": 8416, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36331435499994313, "nonpadding_tokens": 21916, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.36015117399995233, "nonpadding_tokens": 3457, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.35823537999999644, "nonpadding_tokens": 10179, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003925050000361807, "prefetch": {"data_wait_seconds": 0.0012441820001640735, "consumed_batches": 127, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 132, "tokens": 11659993, "loss_tokens": 62468, "loss": 6.085440245405648, "lr": 0.00017489990999999998, "optimizer_lrs": [0.011659994, 0.00017489990999999998], "gradient_norm": 1.1323107481002808, "seconds": 0.287658527000076, "nonpadding_tokens_per_second": 217160.25821123493, "ranks": [{"compute_seconds": 0.2852648799999997, "nonpadding_tokens": 9155, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2789419900000212, "nonpadding_tokens": 2828, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.28221236199999566, "nonpadding_tokens": 6931, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.27819015399995806, "nonpadding_tokens": 5061, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.28128726900001766, "nonpadding_tokens": 4016, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.28374564699993243, "nonpadding_tokens": 10812, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2817155289999391, "nonpadding_tokens": 9762, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.28131550900002367, "nonpadding_tokens": 13903, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003653050000593794, "prefetch": {"data_wait_seconds": 0.0012530900002047929, "consumed_batches": 128, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 133, "tokens": 11743561, "loss_tokens": 83568, "loss": 6.098730225684473, "lr": 0.00017615343, "optimizer_lrs": [0.011743562, 0.00017615343], "gradient_norm": 1.152649998664856, "seconds": 0.34741799800008266, "nonpadding_tokens_per_second": 240540.21519052135, "ranks": [{"compute_seconds": 0.28477172999998857, "nonpadding_tokens": 14113, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2999775480000153, "nonpadding_tokens": 7824, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.29594317900000533, "nonpadding_tokens": 12116, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2986223139999993, "nonpadding_tokens": 5692, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.29668358399999306, "nonpadding_tokens": 2529, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3003644749999239, "nonpadding_tokens": 6772, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2959142070000098, "nonpadding_tokens": 15970, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.29680459299993345, "nonpadding_tokens": 18552, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.03075462599997536, "prefetch": {"data_wait_seconds": 0.0012649050001982687, "consumed_batches": 129, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 134, "tokens": 11829233, "loss_tokens": 85672, "loss": 6.2209954827715, "lr": 0.00017743850999999997, "optimizer_lrs": [0.011829234, 0.00017743850999999997], "gradient_norm": 1.1535736322402954, "seconds": 0.3183430420000377, "nonpadding_tokens_per_second": 269118.49387928465, "ranks": [{"compute_seconds": 0.3137394770000128, "nonpadding_tokens": 12685, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31465919000004305, "nonpadding_tokens": 15423, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31250649200001135, "nonpadding_tokens": 6794, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3148890149999488, "nonpadding_tokens": 2952, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3130264180000495, "nonpadding_tokens": 21550, "peak_allocated_bytes": 14983150080, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31582064199994875, "nonpadding_tokens": 6676, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.312341736999997, "nonpadding_tokens": 8959, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3126741609999044, "nonpadding_tokens": 10633, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004354349999857732, "prefetch": {"data_wait_seconds": 0.001275273000260313, "consumed_batches": 130, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 135, "tokens": 11932633, "loss_tokens": 103400, "loss": 6.10683329303675, "lr": 0.00017898950999999997, "optimizer_lrs": [0.011932634, 0.00017898950999999997], "gradient_norm": 0.888334333896637, "seconds": 0.4336439359999531, "nonpadding_tokens_per_second": 238444.47348621793, "ranks": [{"compute_seconds": 0.43086171300001297, "nonpadding_tokens": 11382, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.42701728899999125, "nonpadding_tokens": 9413, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.42763771399995676, "nonpadding_tokens": 12120, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.43133770100007496, "nonpadding_tokens": 11452, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4291388530000404, "nonpadding_tokens": 11047, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.42659831399998893, "nonpadding_tokens": 22235, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.42762648799998715, "nonpadding_tokens": 3939, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.4297369249999292, "nonpadding_tokens": 21812, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036001600005874934, "prefetch": {"data_wait_seconds": 0.0012870110002722868, "consumed_batches": 131, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 136, "tokens": 12022821, "loss_tokens": 90188, "loss": 6.242049247128221, "lr": 0.00018034232999999998, "optimizer_lrs": [0.012022821999999999, 0.00018034232999999998], "gradient_norm": 0.868883490562439, "seconds": 0.44415174599998863, "nonpadding_tokens_per_second": 203056.72737353668, "ranks": [{"compute_seconds": 0.4397792339999569, "nonpadding_tokens": 10551, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4417736309999327, "nonpadding_tokens": 12292, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4370063900000787, "nonpadding_tokens": 12994, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4416005140000152, "nonpadding_tokens": 12123, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4403899909999609, "nonpadding_tokens": 19499, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4371956370000589, "nonpadding_tokens": 12148, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.44066880399998354, "nonpadding_tokens": 7011, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.44042504399999416, "nonpadding_tokens": 3570, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000381349000008413, "prefetch": {"data_wait_seconds": 0.001297716000294713, "consumed_batches": 132, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 137, "tokens": 12088221, "loss_tokens": 65400, "loss": 5.975523222477064, "lr": 0.00018132332999999998, "optimizer_lrs": [0.012088222, 0.00018132332999999998], "gradient_norm": 1.407638430595398, "seconds": 0.36165267499995934, "nonpadding_tokens_per_second": 180836.48904299506, "ranks": [{"compute_seconds": 0.3582670189999817, "nonpadding_tokens": 8140, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3591611680000142, "nonpadding_tokens": 12416, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35562672399998974, "nonpadding_tokens": 3523, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35494207700003244, "nonpadding_tokens": 6577, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3560024789999261, "nonpadding_tokens": 4030, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3553545119999626, "nonpadding_tokens": 5943, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.35809794300007525, "nonpadding_tokens": 20797, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.35819478199994137, "nonpadding_tokens": 3974, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000576747000081923, "prefetch": {"data_wait_seconds": 0.0013079750002589208, "consumed_batches": 133, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 138, "tokens": 12159649, "loss_tokens": 71428, "loss": 6.048519138153106, "lr": 0.00018239475, "optimizer_lrs": [0.01215965, 0.00018239475], "gradient_norm": 0.9064561128616333, "seconds": 0.33413710900003935, "nonpadding_tokens_per_second": 213768.5341618, "ranks": [{"compute_seconds": 0.33070878600005926, "nonpadding_tokens": 7648, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33059706299991376, "nonpadding_tokens": 2588, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32894841799998176, "nonpadding_tokens": 15197, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3315034490000244, "nonpadding_tokens": 13229, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3303280769999901, "nonpadding_tokens": 4824, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3317349420000255, "nonpadding_tokens": 2581, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3303094829999509, "nonpadding_tokens": 16113, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.3305786209999724, "nonpadding_tokens": 9248, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036611400003039307, "prefetch": {"data_wait_seconds": 0.001317646000188688, "consumed_batches": 134, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 139, "tokens": 12267652, "loss_tokens": 108003, "loss": 6.327517522661408, "lr": 0.000184014795, "optimizer_lrs": [0.012267653000000002, 0.000184014795], "gradient_norm": 0.846211314201355, "seconds": 0.27984626900001786, "nonpadding_tokens_per_second": 385936.8945168717, "ranks": [{"compute_seconds": 0.2746169120000559, "nonpadding_tokens": 5721, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2775489639999478, "nonpadding_tokens": 10791, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.27455371799999284, "nonpadding_tokens": 12658, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2715488359999654, "nonpadding_tokens": 16730, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.27650996700003816, "nonpadding_tokens": 17380, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2763260360000004, "nonpadding_tokens": 12515, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.27300203599997985, "nonpadding_tokens": 15982, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.27598118299999896, "nonpadding_tokens": 16226, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037827300002390984, "prefetch": {"data_wait_seconds": 0.001328633000184709, "consumed_batches": 135, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 140, "tokens": 12378692, "loss_tokens": 111040, "loss": 6.234172370317003, "lr": 0.000185680395, "optimizer_lrs": [0.012378693, 0.000185680395], "gradient_norm": 2.3493285179138184, "seconds": 0.4349418819999755, "nonpadding_tokens_per_second": 255298.47686640176, "ranks": [{"compute_seconds": 0.43026024699997834, "nonpadding_tokens": 25148, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4325367380000671, "nonpadding_tokens": 19752, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4284190939999917, "nonpadding_tokens": 18980, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.42732402399997227, "nonpadding_tokens": 9519, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4308367769999677, "nonpadding_tokens": 5669, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.43254468400004953, "nonpadding_tokens": 17503, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4308418429999392, "nonpadding_tokens": 10512, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.42882176299997354, "nonpadding_tokens": 3957, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003493279999702281, "prefetch": {"data_wait_seconds": 0.001337088000127551, "consumed_batches": 136, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 141, "tokens": 12503171, "loss_tokens": 124479, "loss": 6.159006740092706, "lr": 0.00018754758, "optimizer_lrs": [0.012503172000000002, 0.00018754758], "gradient_norm": 1.0492466688156128, "seconds": 0.404289102000007, "nonpadding_tokens_per_second": 307896.00655621395, "ranks": [{"compute_seconds": 0.3993167969999831, "nonpadding_tokens": 24328, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4019475090000242, "nonpadding_tokens": 4523, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4012178430000404, "nonpadding_tokens": 17445, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.398024962999898, "nonpadding_tokens": 3372, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4005530029999136, "nonpadding_tokens": 15176, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.398212009999952, "nonpadding_tokens": 18963, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4019957240000167, "nonpadding_tokens": 19026, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.40171183699999347, "nonpadding_tokens": 21646, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003492179999966538, "prefetch": {"data_wait_seconds": 0.0013464770001974102, "consumed_batches": 137, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 142, "tokens": 12612922, "loss_tokens": 109751, "loss": 6.069922483622017, "lr": 0.000189193845, "optimizer_lrs": [0.012612923000000002, 0.000189193845], "gradient_norm": 0.893004834651947, "seconds": 0.37903383399998347, "nonpadding_tokens_per_second": 289554.6258807196, "ranks": [{"compute_seconds": 0.376500061999991, "nonpadding_tokens": 10609, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3728706690000081, "nonpadding_tokens": 18738, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.37351001900003666, "nonpadding_tokens": 22351, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37187326200000825, "nonpadding_tokens": 6606, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37610129300003337, "nonpadding_tokens": 16644, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3730860420000681, "nonpadding_tokens": 13014, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3764779749999434, "nonpadding_tokens": 11019, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.376138286000014, "nonpadding_tokens": 10770, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003847230000246782, "prefetch": {"data_wait_seconds": 0.0013553530002354819, "consumed_batches": 138, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 143, "tokens": 12688411, "loss_tokens": 75489, "loss": 5.810429748042761, "lr": 0.00019032617999999998, "optimizer_lrs": [0.012688412, 0.00019032617999999998], "gradient_norm": 1.0893558263778687, "seconds": 0.3305970910000724, "nonpadding_tokens_per_second": 228341.39215091724, "ranks": [{"compute_seconds": 0.327825770000004, "nonpadding_tokens": 10009, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3234885709999844, "nonpadding_tokens": 1277, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3248852450000186, "nonpadding_tokens": 14738, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3278112450000208, "nonpadding_tokens": 4461, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3273590180000383, "nonpadding_tokens": 11742, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32818881500008956, "nonpadding_tokens": 13643, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32767833899993093, "nonpadding_tokens": 777, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.327512246000083, "nonpadding_tokens": 18842, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003962009999440852, "prefetch": {"data_wait_seconds": 0.0013650840002128461, "consumed_batches": 139, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 144, "tokens": 12781743, "loss_tokens": 93332, "loss": 6.247186120516007, "lr": 0.00019172615999999998, "optimizer_lrs": [0.012781744, 0.00019172615999999998], "gradient_norm": 4.169984340667725, "seconds": 0.35218403099997886, "nonpadding_tokens_per_second": 265009.1764098345, "ranks": [{"compute_seconds": 0.3483033830000295, "nonpadding_tokens": 10576, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3457860849999861, "nonpadding_tokens": 2417, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3462879130000829, "nonpadding_tokens": 12129, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34986845399998856, "nonpadding_tokens": 5887, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3463602809999884, "nonpadding_tokens": 12397, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3495085349999272, "nonpadding_tokens": 15162, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3487403799999811, "nonpadding_tokens": 27175, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19656605696}, {"compute_seconds": 0.349467312999991, "nonpadding_tokens": 7589, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040402100000846985, "prefetch": {"data_wait_seconds": 0.0013738980002244716, "consumed_batches": 140, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 145, "tokens": 12898800, "loss_tokens": 117057, "loss": 6.141924980992166, "lr": 0.00019348201499999997, "optimizer_lrs": [0.012898801, 0.00019348201499999997], "gradient_norm": 0.9993845820426941, "seconds": 0.36588073600000826, "nonpadding_tokens_per_second": 319932.12126914866, "ranks": [{"compute_seconds": 0.36214845900008186, "nonpadding_tokens": 16900, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3589462149999463, "nonpadding_tokens": 19435, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35991605200001686, "nonpadding_tokens": 27321, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36352130900002066, "nonpadding_tokens": 8085, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3622380560000238, "nonpadding_tokens": 6155, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3598798059999808, "nonpadding_tokens": 19111, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3623305970000956, "nonpadding_tokens": 8950, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36030273900007614, "nonpadding_tokens": 11100, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00044383899989952624, "prefetch": {"data_wait_seconds": 0.0013840900002151102, "consumed_batches": 141, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 146, "tokens": 13012763, "loss_tokens": 113963, "loss": 6.067527399243614, "lr": 0.00019519146, "optimizer_lrs": [0.013012764000000001, 0.00019519146], "gradient_norm": 1.149950623512268, "seconds": 0.31143813500000306, "nonpadding_tokens_per_second": 365925.0014453075, "ranks": [{"compute_seconds": 0.30685641399998076, "nonpadding_tokens": 4904, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3090570330000446, "nonpadding_tokens": 26184, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3078610600000502, "nonpadding_tokens": 20304, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30649376899998515, "nonpadding_tokens": 3224, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.307857482999907, "nonpadding_tokens": 11858, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3050194999999576, "nonpadding_tokens": 16076, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3062171160000844, "nonpadding_tokens": 6820, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30527519899999334, "nonpadding_tokens": 24593, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004015650000610549, "prefetch": {"data_wait_seconds": 0.0013950910001767625, "consumed_batches": 142, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 147, "tokens": 13111634, "loss_tokens": 98871, "loss": 6.165280390609987, "lr": 0.00019667452499999997, "optimizer_lrs": [0.013111634999999998, 0.00019667452499999997], "gradient_norm": 0.9486029148101807, "seconds": 0.3572775879999881, "nonpadding_tokens_per_second": 276734.4029427429, "ranks": [{"compute_seconds": 0.2943501129999504, "nonpadding_tokens": 8866, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.323639844000013, "nonpadding_tokens": 12422, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3203265490000149, "nonpadding_tokens": 16326, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3236535390000199, "nonpadding_tokens": 10011, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32152444100006505, "nonpadding_tokens": 14474, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31989540199992916, "nonpadding_tokens": 8374, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3192552959999375, "nonpadding_tokens": 11594, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3227747119999549, "nonpadding_tokens": 16804, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.026350779000040347, "prefetch": {"data_wait_seconds": 0.0014047890002757413, "consumed_batches": 143, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 148, "tokens": 13206099, "loss_tokens": 94465, "loss": 6.019781135870428, "lr": 0.0001980915, "optimizer_lrs": [0.0132061, 0.0001980915], "gradient_norm": 1.0701426267623901, "seconds": 0.32860691600001246, "nonpadding_tokens_per_second": 287471.1255316258, "ranks": [{"compute_seconds": 0.3243870840000227, "nonpadding_tokens": 9285, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32087776900004883, "nonpadding_tokens": 17182, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3211332969999603, "nonpadding_tokens": 20509, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.321195012999965, "nonpadding_tokens": 8269, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32248840200009, "nonpadding_tokens": 11550, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.326219401000003, "nonpadding_tokens": 8252, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32200397600001907, "nonpadding_tokens": 3871, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32371366400002444, "nonpadding_tokens": 15547, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035000999992007564, "prefetch": {"data_wait_seconds": 0.0014131980002503042, "consumed_batches": 144, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 149, "tokens": 13280244, "loss_tokens": 74145, "loss": 5.935634145930272, "lr": 0.00019920367499999997, "optimizer_lrs": [0.013280245, 0.00019920367499999997], "gradient_norm": 1.2024184465408325, "seconds": 0.3276334600000155, "nonpadding_tokens_per_second": 226304.72479824405, "ranks": [{"compute_seconds": 0.2958148920000667, "nonpadding_tokens": 10800, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3220444830000133, "nonpadding_tokens": 11858, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3251074659999631, "nonpadding_tokens": 12734, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32205667800008086, "nonpadding_tokens": 15503, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3250120960000231, "nonpadding_tokens": 12939, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3220540400000118, "nonpadding_tokens": 4336, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32326593300001605, "nonpadding_tokens": 2854, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32192530899999383, "nonpadding_tokens": 3121, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004994070000066131, "prefetch": {"data_wait_seconds": 0.0014248350003072119, "consumed_batches": 145, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 150, "tokens": 13374393, "loss_tokens": 94149, "loss": 5.982989065205153, "lr": 0.00020061590999999999, "optimizer_lrs": [0.013374394, 0.00020061590999999999], "gradient_norm": 1.6107100248336792, "seconds": 0.35106867800004693, "nonpadding_tokens_per_second": 268178.29644143704, "ranks": [{"compute_seconds": 0.34571061500003, "nonpadding_tokens": 7895, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3456983379999201, "nonpadding_tokens": 22829, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34563865099994473, "nonpadding_tokens": 10096, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3484899029999724, "nonpadding_tokens": 14183, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3460916389999511, "nonpadding_tokens": 15567, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3454207979999637, "nonpadding_tokens": 11409, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3481860480000023, "nonpadding_tokens": 11176, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3470955969999068, "nonpadding_tokens": 994, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039691500001026725, "prefetch": {"data_wait_seconds": 0.0014348640003163382, "consumed_batches": 146, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 151, "tokens": 13467299, "loss_tokens": 92906, "loss": 5.9222581157298775, "lr": 0.0002020095, "optimizer_lrs": [0.0134673, 0.0002020095], "gradient_norm": 1.0862152576446533, "seconds": 0.4096004760000369, "nonpadding_tokens_per_second": 226821.0254716394, "ranks": [{"compute_seconds": 0.40563535599994793, "nonpadding_tokens": 16159, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40263754600005086, "nonpadding_tokens": 14929, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.40630486500003826, "nonpadding_tokens": 9642, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.40644354599999133, "nonpadding_tokens": 9284, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4052137810000431, "nonpadding_tokens": 14834, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.40715358600004947, "nonpadding_tokens": 16616, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.40382340500002556, "nonpadding_tokens": 8982, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4056756489999316, "nonpadding_tokens": 2460, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040743299996393034, "prefetch": {"data_wait_seconds": 0.0014443570003095374, "consumed_batches": 147, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 152, "tokens": 13534200, "loss_tokens": 66901, "loss": 6.032681219264286, "lr": 0.00020301301499999996, "optimizer_lrs": [0.013534200999999997, 0.00020301301499999996], "gradient_norm": 1.9371742010116577, "seconds": 0.3694256270000551, "nonpadding_tokens_per_second": 181094.63748704694, "ranks": [{"compute_seconds": 0.36652243599996837, "nonpadding_tokens": 15205, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3670668229999592, "nonpadding_tokens": 6753, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36354337799991754, "nonpadding_tokens": 17495, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3669868240000369, "nonpadding_tokens": 1986, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36346451699989757, "nonpadding_tokens": 3279, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3631934509999155, "nonpadding_tokens": 9342, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3663878590000422, "nonpadding_tokens": 6620, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36716951700009304, "nonpadding_tokens": 6221, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036272700003792124, "prefetch": {"data_wait_seconds": 0.0014533660003053228, "consumed_batches": 148, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 153, "tokens": 13611888, "loss_tokens": 77688, "loss": 5.720319981850479, "lr": 0.00020417833499999997, "optimizer_lrs": [0.013611888999999999, 0.00020417833499999997], "gradient_norm": 0.8417879343032837, "seconds": 0.3604568419999623, "nonpadding_tokens_per_second": 215526.4956796357, "ranks": [{"compute_seconds": 0.3526978950000057, "nonpadding_tokens": 17179, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35670311700005186, "nonpadding_tokens": 7448, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3521708320000698, "nonpadding_tokens": 6262, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35215618400002313, "nonpadding_tokens": 2006, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35580727299998216, "nonpadding_tokens": 10548, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3579877450000595, "nonpadding_tokens": 9300, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3545068610000044, "nonpadding_tokens": 11296, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3557969549999598, "nonpadding_tokens": 13649, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003870319999350613, "prefetch": {"data_wait_seconds": 0.0014627130003646016, "consumed_batches": 149, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 154, "tokens": 13737273, "loss_tokens": 125385, "loss": 6.893344997407984, "lr": 0.00020605910999999997, "optimizer_lrs": [0.013737273999999999, 0.00020605910999999997], "gradient_norm": 3.813978910446167, "seconds": 0.46876169500001197, "nonpadding_tokens_per_second": 267481.3265192174, "ranks": [{"compute_seconds": 0.46642700200004583, "nonpadding_tokens": 13846, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4650719159999426, "nonpadding_tokens": 14503, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4650570000000016, "nonpadding_tokens": 15857, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4619771940000419, "nonpadding_tokens": 19380, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4644674420000001, "nonpadding_tokens": 13717, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.46117222800000945, "nonpadding_tokens": 16103, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4649184569999534, "nonpadding_tokens": 21947, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.46500372700006665, "nonpadding_tokens": 10032, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003484610000441535, "prefetch": {"data_wait_seconds": 0.0014714250003180496, "consumed_batches": 150, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 155, "tokens": 13834067, "loss_tokens": 96794, "loss": 6.129761400500031, "lr": 0.00020751101999999996, "optimizer_lrs": [0.013834067999999998, 0.00020751101999999996], "gradient_norm": 1.0939128398895264, "seconds": 0.37336808899999596, "nonpadding_tokens_per_second": 259245.5082576729, "ranks": [{"compute_seconds": 0.36845274100005554, "nonpadding_tokens": 11224, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3706081500000664, "nonpadding_tokens": 4976, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3669445809999843, "nonpadding_tokens": 14533, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3709854149999501, "nonpadding_tokens": 7684, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3692750820000583, "nonpadding_tokens": 19211, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37064621799993347, "nonpadding_tokens": 12514, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3698895499999253, "nonpadding_tokens": 9527, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36863553099999535, "nonpadding_tokens": 17125, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003735789999836925, "prefetch": {"data_wait_seconds": 0.001480073000379889, "consumed_batches": 151, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 156, "tokens": 13949629, "loss_tokens": 115562, "loss": 5.873772304044582, "lr": 0.00020924444999999997, "optimizer_lrs": [0.01394963, 0.00020924444999999997], "gradient_norm": 2.565274953842163, "seconds": 0.3272360570000501, "nonpadding_tokens_per_second": 353145.68039787345, "ranks": [{"compute_seconds": 0.32299696600000516, "nonpadding_tokens": 16403, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3157002929999635, "nonpadding_tokens": 22417, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.316650568, "nonpadding_tokens": 7709, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3194996380000248, "nonpadding_tokens": 8512, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.319065941999952, "nonpadding_tokens": 14629, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31984507499998927, "nonpadding_tokens": 5692, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32493453100005354, "nonpadding_tokens": 10069, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31685939999999846, "nonpadding_tokens": 30131, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035301900004469644, "prefetch": {"data_wait_seconds": 0.001489901000354621, "consumed_batches": 152, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 157, "tokens": 14031056, "loss_tokens": 81427, "loss": 5.6889898313827105, "lr": 0.00021046585499999998, "optimizer_lrs": [0.014031057, 0.00021046585499999998], "gradient_norm": 1.4182050228118896, "seconds": 0.3034933540000111, "nonpadding_tokens_per_second": 268299.1206456436, "ranks": [{"compute_seconds": 0.29969923600003767, "nonpadding_tokens": 2200, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3000723149999658, "nonpadding_tokens": 15328, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.29903603299999304, "nonpadding_tokens": 13630, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.29983357100002195, "nonpadding_tokens": 10465, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2986002750000125, "nonpadding_tokens": 7357, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3011185300000534, "nonpadding_tokens": 12798, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2990322909998895, "nonpadding_tokens": 11376, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2984322390000216, "nonpadding_tokens": 8273, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003937669999913851, "prefetch": {"data_wait_seconds": 0.0014980960004322696, "consumed_batches": 153, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 158, "tokens": 14121608, "loss_tokens": 90552, "loss": 6.171271755455429, "lr": 0.00021182413499999997, "optimizer_lrs": [0.014121608999999999, 0.00021182413499999997], "gradient_norm": 0.9863476157188416, "seconds": 0.3335867949999738, "nonpadding_tokens_per_second": 271449.59380063927, "ranks": [{"compute_seconds": 0.3288340320000316, "nonpadding_tokens": 18651, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3304615690000219, "nonpadding_tokens": 9658, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32668698899999526, "nonpadding_tokens": 12678, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33010015399997883, "nonpadding_tokens": 8022, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3270243949999667, "nonpadding_tokens": 14864, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33109317599996757, "nonpadding_tokens": 7001, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32864963700001226, "nonpadding_tokens": 4533, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32818038599998545, "nonpadding_tokens": 15145, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003946859999359731, "prefetch": {"data_wait_seconds": 0.0015076640004281217, "consumed_batches": 154, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 159, "tokens": 14211549, "loss_tokens": 89941, "loss": 5.933591187556287, "lr": 0.00021317324999999996, "optimizer_lrs": [0.014211549999999998, 0.00021317324999999996], "gradient_norm": 0.9021503925323486, "seconds": 0.28572827400000733, "nonpadding_tokens_per_second": 314778.08877954335, "ranks": [{"compute_seconds": 0.2808541710001009, "nonpadding_tokens": 2494, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2792047130000128, "nonpadding_tokens": 11640, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.28333188900001005, "nonpadding_tokens": 18835, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2798217399999885, "nonpadding_tokens": 784, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2831045970000332, "nonpadding_tokens": 3632, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.28018971400001647, "nonpadding_tokens": 16731, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.28238624899995557, "nonpadding_tokens": 10860, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.280028879999918, "nonpadding_tokens": 24965, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040376500010097516, "prefetch": {"data_wait_seconds": 0.0015178070004822075, "consumed_batches": 155, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 160, "tokens": 14276484, "loss_tokens": 64935, "loss": 5.4146521521521525, "lr": 0.000214147275, "optimizer_lrs": [0.014276485, 0.000214147275], "gradient_norm": 1.1328824758529663, "seconds": 0.3221714300000258, "nonpadding_tokens_per_second": 201554.18498777127, "ranks": [{"compute_seconds": 0.3188309780000509, "nonpadding_tokens": 7866, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31953296999995473, "nonpadding_tokens": 9676, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31592683700000634, "nonpadding_tokens": 2324, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31967157700000826, "nonpadding_tokens": 8259, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31841602300005434, "nonpadding_tokens": 11441, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3197503810000626, "nonpadding_tokens": 13672, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3163669680000112, "nonpadding_tokens": 2218, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3184289929999977, "nonpadding_tokens": 9479, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003784779999023158, "prefetch": {"data_wait_seconds": 0.0015267410004753401, "consumed_batches": 156, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 161, "tokens": 14338435, "loss_tokens": 61951, "loss": 5.950391741053413, "lr": 0.00021507654, "optimizer_lrs": [0.014338436000000001, 0.00021507654], "gradient_norm": 1.364776611328125, "seconds": 0.2864376769999808, "nonpadding_tokens_per_second": 216280.9049732803, "ranks": [{"compute_seconds": 0.28272103800009063, "nonpadding_tokens": 5954, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.283488540999997, "nonpadding_tokens": 9613, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.27889850200006094, "nonpadding_tokens": 2150, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.28405916799999886, "nonpadding_tokens": 12255, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2810199549999197, "nonpadding_tokens": 2340, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2837879479999401, "nonpadding_tokens": 7674, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2823248290000038, "nonpadding_tokens": 11943, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2818231389999255, "nonpadding_tokens": 10022, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003929279999965729, "prefetch": {"data_wait_seconds": 0.0015366330004553674, "consumed_batches": 157, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 162, "tokens": 14405719, "loss_tokens": 67284, "loss": 5.885212773467689, "lr": 0.00021608579999999998, "optimizer_lrs": [0.01440572, 0.00021608579999999998], "gradient_norm": 0.8593166470527649, "seconds": 0.34293758100000105, "nonpadding_tokens_per_second": 196198.9695145129, "ranks": [{"compute_seconds": 0.336141800000064, "nonpadding_tokens": 11256, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33577680000007604, "nonpadding_tokens": 11631, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33537240600003315, "nonpadding_tokens": 14562, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3354390900000226, "nonpadding_tokens": 1390, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33753779699998177, "nonpadding_tokens": 3331, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.340468278000003, "nonpadding_tokens": 17751, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33676272399998197, "nonpadding_tokens": 3561, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3363505930000201, "nonpadding_tokens": 3802, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00042534299996077607, "prefetch": {"data_wait_seconds": 0.001546827000538542, "consumed_batches": 158, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 163, "tokens": 14488914, "loss_tokens": 83195, "loss": 5.958674649918866, "lr": 0.00021733372499999998, "optimizer_lrs": [0.014488915, 0.00021733372499999998], "gradient_norm": 0.9071171283721924, "seconds": 0.43993244799992226, "nonpadding_tokens_per_second": 189108.57877892812, "ranks": [{"compute_seconds": 0.432623299999932, "nonpadding_tokens": 16816, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.43207118099996933, "nonpadding_tokens": 2428, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4320819740000843, "nonpadding_tokens": 12850, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.43476572499992017, "nonpadding_tokens": 7466, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4328515969999671, "nonpadding_tokens": 17901, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.437338676999957, "nonpadding_tokens": 10356, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4331241970000974, "nonpadding_tokens": 6209, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.43413756899997225, "nonpadding_tokens": 9169, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003741949999493954, "prefetch": {"data_wait_seconds": 0.0015564310004947401, "consumed_batches": 159, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 164, "tokens": 14554112, "loss_tokens": 65198, "loss": 5.642594002116629, "lr": 0.00021831169499999996, "optimizer_lrs": [0.014554112999999999, 0.00021831169499999996], "gradient_norm": 1.151404619216919, "seconds": 0.35782959100004064, "nonpadding_tokens_per_second": 182204.04807156543, "ranks": [{"compute_seconds": 0.3540972380000085, "nonpadding_tokens": 3548, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.354409851000014, "nonpadding_tokens": 10355, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.351062211999988, "nonpadding_tokens": 4994, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35044974800007367, "nonpadding_tokens": 4614, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35141705700004877, "nonpadding_tokens": 11595, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35535325299997567, "nonpadding_tokens": 7015, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3536950939999315, "nonpadding_tokens": 15425, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3533533449999595, "nonpadding_tokens": 7652, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036793900005704927, "prefetch": {"data_wait_seconds": 0.0015652040004852097, "consumed_batches": 160, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 165, "tokens": 14622339, "loss_tokens": 68227, "loss": 5.970870678030692, "lr": 0.00021933509999999998, "optimizer_lrs": [0.01462234, 0.00021933509999999998], "gradient_norm": 1.2130422592163086, "seconds": 0.3680391159999772, "nonpadding_tokens_per_second": 185379.75186312594, "ranks": [{"compute_seconds": 0.3643893340000659, "nonpadding_tokens": 6603, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.36518752300003143, "nonpadding_tokens": 3873, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36435205199995835, "nonpadding_tokens": 4358, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.364316461000044, "nonpadding_tokens": 12984, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36098923800000193, "nonpadding_tokens": 12783, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.365639715000043, "nonpadding_tokens": 4530, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3641668379999601, "nonpadding_tokens": 10886, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36380172399992716, "nonpadding_tokens": 12210, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003495339999517455, "prefetch": {"data_wait_seconds": 0.0015748130005022176, "consumed_batches": 161, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 166, "tokens": 14706857, "loss_tokens": 84518, "loss": 6.056622553775527, "lr": 0.00022060287, "optimizer_lrs": [0.014706858, 0.00022060287], "gradient_norm": 1.157010793685913, "seconds": 0.4099788390000185, "nonpadding_tokens_per_second": 206152.10337720913, "ranks": [{"compute_seconds": 0.40566217699995377, "nonpadding_tokens": 11203, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40760693400000036, "nonpadding_tokens": 3453, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4038662140000042, "nonpadding_tokens": 8380, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4044457270000521, "nonpadding_tokens": 11895, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4060092419999819, "nonpadding_tokens": 10145, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4070126200000459, "nonpadding_tokens": 18191, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.40457544899993536, "nonpadding_tokens": 10453, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.40641806999997243, "nonpadding_tokens": 10798, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003839039999320448, "prefetch": {"data_wait_seconds": 0.0015857090005511054, "consumed_batches": 162, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 167, "tokens": 14821355, "loss_tokens": 114498, "loss": 5.696709767856207, "lr": 0.00022232034, "optimizer_lrs": [0.014821356, 0.00022232034], "gradient_norm": 0.8287169337272644, "seconds": 0.31203457200001594, "nonpadding_tokens_per_second": 366940.10944400786, "ranks": [{"compute_seconds": 0.2895676830000866, "nonpadding_tokens": 9182, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.29752035100000285, "nonpadding_tokens": 9195, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2963484240000298, "nonpadding_tokens": 19711, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2973853330000793, "nonpadding_tokens": 9294, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.29589120100001765, "nonpadding_tokens": 8737, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.29338607799991223, "nonpadding_tokens": 22235, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.29461477500001365, "nonpadding_tokens": 13009, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2942208129999244, "nonpadding_tokens": 23135, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003854990000036196, "prefetch": {"data_wait_seconds": 0.0015949590006130165, "consumed_batches": 163, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 168, "tokens": 14949127, "loss_tokens": 127772, "loss": 6.069527165576183, "lr": 0.00022423691999999998, "optimizer_lrs": [0.014949127999999999, 0.00022423691999999998], "gradient_norm": 0.8912922143936157, "seconds": 0.31045744000005016, "nonpadding_tokens_per_second": 411560.4380425844, "ranks": [{"compute_seconds": 0.3062533270000358, "nonpadding_tokens": 1697, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3035998519999339, "nonpadding_tokens": 32545, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3045644149999589, "nonpadding_tokens": 4721, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30816134100007275, "nonpadding_tokens": 27804, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30358560100000886, "nonpadding_tokens": 20719, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3037053210000522, "nonpadding_tokens": 7373, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.30722692500000903, "nonpadding_tokens": 12820, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30701728599990474, "nonpadding_tokens": 20093, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003519110000524961, "prefetch": {"data_wait_seconds": 0.001603862000592926, "consumed_batches": 164, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 169, "tokens": 15057161, "loss_tokens": 108034, "loss": 5.626793416887276, "lr": 0.00022585742999999998, "optimizer_lrs": [0.015057161999999999, 0.00022585742999999998], "gradient_norm": 0.9769420623779297, "seconds": 0.3504797220000455, "nonpadding_tokens_per_second": 308246.0787845123, "ranks": [{"compute_seconds": 0.3458607119999897, "nonpadding_tokens": 22396, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34589178600003834, "nonpadding_tokens": 11485, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3453951930000585, "nonpadding_tokens": 6428, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3455752400000165, "nonpadding_tokens": 7683, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3482026510000651, "nonpadding_tokens": 28311, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3445625589999963, "nonpadding_tokens": 16345, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3471214859999918, "nonpadding_tokens": 10463, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34751334299994596, "nonpadding_tokens": 4923, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000380156000005627, "prefetch": {"data_wait_seconds": 0.0016137650005703108, "consumed_batches": 165, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 170, "tokens": 15166448, "loss_tokens": 109287, "loss": 5.774503143100278, "lr": 0.00022749673499999998, "optimizer_lrs": [0.015166449, 0.00022749673499999998], "gradient_norm": 0.9311195611953735, "seconds": 0.42603234499995324, "nonpadding_tokens_per_second": 256522.77645729456, "ranks": [{"compute_seconds": 0.4223377980000578, "nonpadding_tokens": 11051, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4236671750001051, "nonpadding_tokens": 13047, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4201588639999727, "nonpadding_tokens": 17365, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4235563160000311, "nonpadding_tokens": 16895, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.42236305400001584, "nonpadding_tokens": 11474, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.41952221899998676, "nonpadding_tokens": 17908, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4229469469999003, "nonpadding_tokens": 12529, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.41898870700003954, "nonpadding_tokens": 9018, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037373400004980795, "prefetch": {"data_wait_seconds": 0.001623402000632268, "consumed_batches": 166, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 171, "tokens": 15258945, "loss_tokens": 92497, "loss": 5.724189027752251, "lr": 0.00022888418999999997, "optimizer_lrs": [0.015258945999999999, 0.00022888418999999997], "gradient_norm": 1.07954740524292, "seconds": 0.3730186129999993, "nonpadding_tokens_per_second": 247968.85939844555, "ranks": [{"compute_seconds": 0.3707876869999609, "nonpadding_tokens": 14236, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.36738382700002603, "nonpadding_tokens": 5990, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3667604959999835, "nonpadding_tokens": 13915, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37073653200002354, "nonpadding_tokens": 9057, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3696234550000099, "nonpadding_tokens": 4294, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36597970100001476, "nonpadding_tokens": 17870, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3701086779999514, "nonpadding_tokens": 10781, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36824040800001967, "nonpadding_tokens": 16354, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003502280000020619, "prefetch": {"data_wait_seconds": 0.0016324850006412817, "consumed_batches": 167, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 172, "tokens": 15352859, "loss_tokens": 93914, "loss": 6.126907995613007, "lr": 0.00023029289999999998, "optimizer_lrs": [0.01535286, 0.00023029289999999998], "gradient_norm": 1.9088735580444336, "seconds": 0.3387821600000507, "nonpadding_tokens_per_second": 277210.58275319444, "ranks": [{"compute_seconds": 0.33147612900006607, "nonpadding_tokens": 6659, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3352720860000318, "nonpadding_tokens": 7946, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3339288290000013, "nonpadding_tokens": 19108, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3353952569999592, "nonpadding_tokens": 11817, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3331799329999967, "nonpadding_tokens": 13459, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3361091750000469, "nonpadding_tokens": 18452, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3313953690000062, "nonpadding_tokens": 9269, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3332090330000028, "nonpadding_tokens": 7204, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0006142390000150044, "prefetch": {"data_wait_seconds": 0.0016458120006745958, "consumed_batches": 168, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 173, "tokens": 15426698, "loss_tokens": 73839, "loss": 5.304922703449397, "lr": 0.00023140048499999997, "optimizer_lrs": [0.015426698999999999, 0.00023140048499999997], "gradient_norm": 1.1684961318969727, "seconds": 0.4125161999999136, "nonpadding_tokens_per_second": 178996.606678757, "ranks": [{"compute_seconds": 0.4057219060000534, "nonpadding_tokens": 16714, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4094310499999665, "nonpadding_tokens": 10453, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4083277399998906, "nonpadding_tokens": 13070, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4057954529999961, "nonpadding_tokens": 8182, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.40731349899999714, "nonpadding_tokens": 4673, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4100453550000793, "nonpadding_tokens": 4340, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4070304760000454, "nonpadding_tokens": 6348, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4084450970000262, "nonpadding_tokens": 10059, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038468200000352226, "prefetch": {"data_wait_seconds": 0.0016556180006546128, "consumed_batches": 169, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 174, "tokens": 15549008, "loss_tokens": 122310, "loss": 5.893476616793394, "lr": 0.000233235135, "optimizer_lrs": [0.015549009, 0.000233235135], "gradient_norm": 1.023740530014038, "seconds": 0.2805115929999147, "nonpadding_tokens_per_second": 436024.75994650676, "ranks": [{"compute_seconds": 0.2595723550000457, "nonpadding_tokens": 8636, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.26729437900007724, "nonpadding_tokens": 11050, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2653827909999791, "nonpadding_tokens": 16180, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2676264970000375, "nonpadding_tokens": 5350, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2652336150000565, "nonpadding_tokens": 11914, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2627251659999956, "nonpadding_tokens": 25244, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.26413987000000816, "nonpadding_tokens": 18168, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2660120499999721, "nonpadding_tokens": 25768, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004705100000137463, "prefetch": {"data_wait_seconds": 0.001664899000729747, "consumed_batches": 170, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 175, "tokens": 15655557, "loss_tokens": 106549, "loss": 5.912813001529813, "lr": 0.00023483336999999998, "optimizer_lrs": [0.015655558, 0.00023483336999999998], "gradient_norm": 1.0824534893035889, "seconds": 0.3372530139999981, "nonpadding_tokens_per_second": 315931.9430129707, "ranks": [{"compute_seconds": 0.33449601799998163, "nonpadding_tokens": 10012, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3349511440000015, "nonpadding_tokens": 7728, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33195996500001, "nonpadding_tokens": 10255, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33499651700003596, "nonpadding_tokens": 9222, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33294663699996363, "nonpadding_tokens": 18225, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33368450900002244, "nonpadding_tokens": 23601, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33356575699997393, "nonpadding_tokens": 18699, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33228715400002784, "nonpadding_tokens": 8807, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003459099999645332, "prefetch": {"data_wait_seconds": 0.0016737420007757464, "consumed_batches": 171, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 176, "tokens": 15751308, "loss_tokens": 95751, "loss": 5.789784179799689, "lr": 0.00023626963499999998, "optimizer_lrs": [0.015751309, 0.00023626963499999998], "gradient_norm": 1.3320220708847046, "seconds": 0.2691600499999822, "nonpadding_tokens_per_second": 355740.014166316, "ranks": [{"compute_seconds": 0.2634694439999521, "nonpadding_tokens": 7285, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2649327899999889, "nonpadding_tokens": 4571, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.26237270500007526, "nonpadding_tokens": 14194, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.26590233299998545, "nonpadding_tokens": 15604, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2647895589999507, "nonpadding_tokens": 12335, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.26669551999998475, "nonpadding_tokens": 21361, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2647253650000039, "nonpadding_tokens": 8054, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2647473029999219, "nonpadding_tokens": 12347, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040486800003236567, "prefetch": {"data_wait_seconds": 0.0016834010007187317, "consumed_batches": 172, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 177, "tokens": 15849579, "loss_tokens": 98271, "loss": 5.706254897172106, "lr": 0.0002377437, "optimizer_lrs": [0.015849580000000002, 0.0002377437], "gradient_norm": 1.0342812538146973, "seconds": 0.3593106580000267, "nonpadding_tokens_per_second": 273498.7059582093, "ranks": [{"compute_seconds": 0.3533106160000443, "nonpadding_tokens": 5508, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35216352099996584, "nonpadding_tokens": 13672, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3522676960000126, "nonpadding_tokens": 17278, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35226956200006043, "nonpadding_tokens": 5371, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3547777200000155, "nonpadding_tokens": 10050, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35688635399992563, "nonpadding_tokens": 20673, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.35431707499992626, "nonpadding_tokens": 12477, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35520921799991356, "nonpadding_tokens": 13242, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036368700000366516, "prefetch": {"data_wait_seconds": 0.001693240000690821, "consumed_batches": 173, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 178, "tokens": 15923794, "loss_tokens": 74215, "loss": 5.702243060702014, "lr": 0.00023885692499999998, "optimizer_lrs": [0.015923795, 0.00023885692499999998], "gradient_norm": 1.1155802011489868, "seconds": 0.4613891409999269, "nonpadding_tokens_per_second": 160851.20650902306, "ranks": [{"compute_seconds": 0.4262377219999962, "nonpadding_tokens": 22689, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.42796282599999813, "nonpadding_tokens": 4969, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4273355290000609, "nonpadding_tokens": 11515, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.45833423600004153, "nonpadding_tokens": 10898, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.45829410299995743, "nonpadding_tokens": 3590, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.42835178099994664, "nonpadding_tokens": 7243, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.42695098699994105, "nonpadding_tokens": 3424, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4252021779999495, "nonpadding_tokens": 9887, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034438599993791286, "prefetch": {"data_wait_seconds": 0.0017020080007341676, "consumed_batches": 174, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 179, "tokens": 16000974, "loss_tokens": 77180, "loss": 5.8825946650686705, "lr": 0.00024001462499999998, "optimizer_lrs": [0.016000975, 0.00024001462499999998], "gradient_norm": 1.3190524578094482, "seconds": 0.32032805699998335, "nonpadding_tokens_per_second": 240940.4930770832, "ranks": [{"compute_seconds": 0.31645464999996875, "nonpadding_tokens": 2296, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3175438600000007, "nonpadding_tokens": 19609, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31411647099992024, "nonpadding_tokens": 2108, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3175662899999452, "nonpadding_tokens": 5110, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3144159789999321, "nonpadding_tokens": 24817, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31783673099994303, "nonpadding_tokens": 9842, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3166341690000536, "nonpadding_tokens": 8255, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31622457900004974, "nonpadding_tokens": 5143, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00044779900008506957, "prefetch": {"data_wait_seconds": 0.0017133130007778163, "consumed_batches": 175, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 180, "tokens": 16072484, "loss_tokens": 71510, "loss": 5.531902880715983, "lr": 0.000241087275, "optimizer_lrs": [0.016072485, 0.000241087275], "gradient_norm": 1.9982675313949585, "seconds": 0.33758679799996116, "nonpadding_tokens_per_second": 211827.00396953386, "ranks": [{"compute_seconds": 0.33094605099995533, "nonpadding_tokens": 14757, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33442657700004474, "nonpadding_tokens": 14072, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33358335299999453, "nonpadding_tokens": 13739, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3344655219999595, "nonpadding_tokens": 9490, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3332615079999641, "nonpadding_tokens": 10798, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3351601069999788, "nonpadding_tokens": 1358, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3314054170000418, "nonpadding_tokens": 5296, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.330337770999904, "nonpadding_tokens": 2000, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00041964599995480967, "prefetch": {"data_wait_seconds": 0.0017244660007236234, "consumed_batches": 176, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 181, "tokens": 16175157, "loss_tokens": 102673, "loss": 5.942074596047647, "lr": 0.00024262737, "optimizer_lrs": [0.016175158000000002, 0.00024262737], "gradient_norm": 0.8963872790336609, "seconds": 0.2542884719999847, "nonpadding_tokens_per_second": 403765.84590121004, "ranks": [{"compute_seconds": 0.24790637700004936, "nonpadding_tokens": 12064, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.25072387899990645, "nonpadding_tokens": 13967, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2478950879999502, "nonpadding_tokens": 13720, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.25147757899992484, "nonpadding_tokens": 6738, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2470920840000872, "nonpadding_tokens": 17452, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.25197581399993396, "nonpadding_tokens": 13319, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.250659953999957, "nonpadding_tokens": 4413, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.24765283700003238, "nonpadding_tokens": 21000, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003538949999892793, "prefetch": {"data_wait_seconds": 0.0017331640007114402, "consumed_batches": 177, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 182, "tokens": 16275457, "loss_tokens": 100300, "loss": 5.7835593220338986, "lr": 0.00024413187, "optimizer_lrs": [0.016275458, 0.00024413187], "gradient_norm": 0.8370051383972168, "seconds": 0.35810445799995705, "nonpadding_tokens_per_second": 280085.87371456856, "ranks": [{"compute_seconds": 0.3529375670000263, "nonpadding_tokens": 19333, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3553091080000286, "nonpadding_tokens": 5032, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35126550500001485, "nonpadding_tokens": 17889, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3557974779999995, "nonpadding_tokens": 15986, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3546546869999929, "nonpadding_tokens": 4453, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3518459530000655, "nonpadding_tokens": 10560, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.35467884599995614, "nonpadding_tokens": 7436, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3521951310000304, "nonpadding_tokens": 19611, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036330799991901586, "prefetch": {"data_wait_seconds": 0.001741116000744114, "consumed_batches": 178, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 183, "tokens": 16374709, "loss_tokens": 99252, "loss": 5.650909427517833, "lr": 0.00024562064999999995, "optimizer_lrs": [0.016374709999999997, 0.00024562064999999995], "gradient_norm": 1.035373330116272, "seconds": 0.3186200989999861, "nonpadding_tokens_per_second": 311505.7722708332, "ranks": [{"compute_seconds": 0.30772462899994935, "nonpadding_tokens": 10904, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3086874440000429, "nonpadding_tokens": 14777, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3085006589999466, "nonpadding_tokens": 4772, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3126364959999819, "nonpadding_tokens": 14530, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3087982510000984, "nonpadding_tokens": 10652, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30901845799996863, "nonpadding_tokens": 15882, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31208351399993717, "nonpadding_tokens": 13330, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3119257700000162, "nonpadding_tokens": 14405, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000353915000005145, "prefetch": {"data_wait_seconds": 0.001750140000808642, "consumed_batches": 179, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 184, "tokens": 16466411, "loss_tokens": 91702, "loss": 5.515072940066738, "lr": 0.00024699617999999996, "optimizer_lrs": [0.016466412, 0.00024699617999999996], "gradient_norm": 0.8151354789733887, "seconds": 0.37500177800006895, "nonpadding_tokens_per_second": 244537.5072327874, "ranks": [{"compute_seconds": 0.3711018230000036, "nonpadding_tokens": 12714, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3726807820000886, "nonpadding_tokens": 29184, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36920632499993644, "nonpadding_tokens": 6573, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3724838099999488, "nonpadding_tokens": 7383, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.368466986000044, "nonpadding_tokens": 10068, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37226092900004915, "nonpadding_tokens": 5207, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3717883699999902, "nonpadding_tokens": 12941, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3688150069999665, "nonpadding_tokens": 7632, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037663399996290536, "prefetch": {"data_wait_seconds": 0.0017584270008228486, "consumed_batches": 180, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 185, "tokens": 16525682, "loss_tokens": 59271, "loss": 5.555075416308144, "lr": 0.000247885245, "optimizer_lrs": [0.016525683, 0.000247885245], "gradient_norm": 1.0288952589035034, "seconds": 0.28207934800002477, "nonpadding_tokens_per_second": 210121.727876352, "ranks": [{"compute_seconds": 0.2785253309999689, "nonpadding_tokens": 3442, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2798363369998924, "nonpadding_tokens": 12485, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.27600330500001746, "nonpadding_tokens": 4134, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.27545509000003676, "nonpadding_tokens": 4666, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2786927580000338, "nonpadding_tokens": 4193, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.27590728299992406, "nonpadding_tokens": 23774, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2790326299999606, "nonpadding_tokens": 2353, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.27635568999994575, "nonpadding_tokens": 4224, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003429430000778666, "prefetch": {"data_wait_seconds": 0.0017672130007895248, "consumed_batches": 181, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 186, "tokens": 16605807, "loss_tokens": 80125, "loss": 5.584301092043682, "lr": 0.00024908711999999995, "optimizer_lrs": [0.016605807999999996, 0.00024908711999999995], "gradient_norm": 1.2541916370391846, "seconds": 0.44645023700002184, "nonpadding_tokens_per_second": 179471.29010034792, "ranks": [{"compute_seconds": 0.37417185100002825, "nonpadding_tokens": 20157, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40096601599998394, "nonpadding_tokens": 9595, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3978318910000098, "nonpadding_tokens": 15024, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.39730288600003405, "nonpadding_tokens": 6193, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39987283900006787, "nonpadding_tokens": 4385, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.40203741999994236, "nonpadding_tokens": 6899, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4004664019998927, "nonpadding_tokens": 9732, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.40004537699996945, "nonpadding_tokens": 8140, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.02635582000004888, "prefetch": {"data_wait_seconds": 0.0017770360008171338, "consumed_batches": 182, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 187, "tokens": 16702902, "loss_tokens": 97095, "loss": 5.376850314125341, "lr": 0.000250543545, "optimizer_lrs": [0.016702903, 0.000250543545], "gradient_norm": 1.9005382061004639, "seconds": 0.4329002599999967, "nonpadding_tokens_per_second": 224289.53958124382, "ranks": [{"compute_seconds": 0.39526866599999266, "nonpadding_tokens": 9596, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40335417599999346, "nonpadding_tokens": 4100, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.40409943099996326, "nonpadding_tokens": 925, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.40666390399997, "nonpadding_tokens": 19522, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.40427526100006617, "nonpadding_tokens": 14802, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4033581130000812, "nonpadding_tokens": 11183, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4055157819999522, "nonpadding_tokens": 22155, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4047998539999753, "nonpadding_tokens": 14812, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004931189999979324, "prefetch": {"data_wait_seconds": 0.0017865420009002264, "consumed_batches": 183, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 188, "tokens": 16769980, "loss_tokens": 67078, "loss": 5.4728827819851515, "lr": 0.00025154971499999996, "optimizer_lrs": [0.016769981, 0.00025154971499999996], "gradient_norm": 0.8381584882736206, "seconds": 0.31736168899999484, "nonpadding_tokens_per_second": 211361.3656751149, "ranks": [{"compute_seconds": 0.31126084300001367, "nonpadding_tokens": 6073, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31217934599999353, "nonpadding_tokens": 3010, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30825168900003064, "nonpadding_tokens": 2341, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3119339040000568, "nonpadding_tokens": 12891, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31056631199999174, "nonpadding_tokens": 11310, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3149169640000764, "nonpadding_tokens": 8750, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3107272679999369, "nonpadding_tokens": 8292, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31033912899999905, "nonpadding_tokens": 14411, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036495300003025477, "prefetch": {"data_wait_seconds": 0.0017961340009833293, "consumed_batches": 184, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 189, "tokens": 16880798, "loss_tokens": 110818, "loss": 5.478331588731073, "lr": 0.00025321198499999995, "optimizer_lrs": [0.016880799, 0.00025321198499999995], "gradient_norm": 1.0285351276397705, "seconds": 0.34360065399994255, "nonpadding_tokens_per_second": 322519.758649873, "ranks": [{"compute_seconds": 0.3411934050000127, "nonpadding_tokens": 19961, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3367679140000064, "nonpadding_tokens": 18355, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33845473999997466, "nonpadding_tokens": 9249, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33743090500001927, "nonpadding_tokens": 20245, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3405154299999822, "nonpadding_tokens": 7037, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3381286230001024, "nonpadding_tokens": 18780, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3408598409999968, "nonpadding_tokens": 13818, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34100705199989534, "nonpadding_tokens": 3373, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035358900004212046, "prefetch": {"data_wait_seconds": 0.0018066990010083828, "consumed_batches": 185, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 190, "tokens": 16960544, "loss_tokens": 79746, "loss": 5.535547237478996, "lr": 0.00025440817499999997, "optimizer_lrs": [0.016960545, 0.00025440817499999997], "gradient_norm": 0.9045189023017883, "seconds": 0.3519659400000137, "nonpadding_tokens_per_second": 226573.059881865, "ranks": [{"compute_seconds": 0.34919021100006375, "nonpadding_tokens": 8138, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.345063032999974, "nonpadding_tokens": 3756, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34555273499995565, "nonpadding_tokens": 10954, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34959275699998216, "nonpadding_tokens": 10853, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34613721499999883, "nonpadding_tokens": 10829, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34557731300003525, "nonpadding_tokens": 12368, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.346669881999901, "nonpadding_tokens": 2969, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34554375200002596, "nonpadding_tokens": 19879, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036376799994286557, "prefetch": {"data_wait_seconds": 0.0018178210009409668, "consumed_batches": 186, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 191, "tokens": 17075558, "loss_tokens": 115014, "loss": 5.4495246230893635, "lr": 0.000256133385, "optimizer_lrs": [0.017075559, 0.000256133385], "gradient_norm": 1.1129963397979736, "seconds": 0.5466972140000053, "nonpadding_tokens_per_second": 210379.70755051056, "ranks": [{"compute_seconds": 0.5422327280000445, "nonpadding_tokens": 10415, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.5409097629999451, "nonpadding_tokens": 15774, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.5433424049999758, "nonpadding_tokens": 14130, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.5408600810000053, "nonpadding_tokens": 25541, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.5417639459999464, "nonpadding_tokens": 16567, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.5408776630000602, "nonpadding_tokens": 8227, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.5443581479999011, "nonpadding_tokens": 10291, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.5430874080000194, "nonpadding_tokens": 14069, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038582900003802933, "prefetch": {"data_wait_seconds": 0.0018281620009474864, "consumed_batches": 187, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 192, "tokens": 17164934, "loss_tokens": 89376, "loss": 5.345173061000716, "lr": 0.00025747402499999995, "optimizer_lrs": [0.017164935, 0.00025747402499999995], "gradient_norm": 0.9163107872009277, "seconds": 0.3093136789999562, "nonpadding_tokens_per_second": 288949.3936671732, "ranks": [{"compute_seconds": 0.3063469350000787, "nonpadding_tokens": 25134, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3028830730000891, "nonpadding_tokens": 9184, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3034467649999897, "nonpadding_tokens": 12148, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30690725800002383, "nonpadding_tokens": 3512, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30553187600003184, "nonpadding_tokens": 10731, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3024235079999471, "nonpadding_tokens": 7355, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3057388270000274, "nonpadding_tokens": 9001, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.303866591999963, "nonpadding_tokens": 12311, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034410399996431806, "prefetch": {"data_wait_seconds": 0.0018375140009538882, "consumed_batches": 188, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 193, "tokens": 17226928, "loss_tokens": 61994, "loss": 5.63181114301384, "lr": 0.000258403935, "optimizer_lrs": [0.017226929000000002, 0.000258403935], "gradient_norm": 1.0335438251495361, "seconds": 0.3106340709999813, "nonpadding_tokens_per_second": 199572.4416205579, "ranks": [{"compute_seconds": 0.30804281200005335, "nonpadding_tokens": 4506, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30382657499990273, "nonpadding_tokens": 4321, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30680262999999286, "nonpadding_tokens": 7520, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3079952840000715, "nonpadding_tokens": 3044, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3064114230000996, "nonpadding_tokens": 13875, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3036388519999491, "nonpadding_tokens": 1198, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3068278820001069, "nonpadding_tokens": 13104, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3039203189999853, "nonpadding_tokens": 14426, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004117419999829508, "prefetch": {"data_wait_seconds": 0.0018456310009469235, "consumed_batches": 189, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 194, "tokens": 17312914, "loss_tokens": 85986, "loss": 5.761697398413696, "lr": 0.000259693725, "optimizer_lrs": [0.017312915, 0.000259693725], "gradient_norm": 0.8265958428382874, "seconds": 0.3834166510000614, "nonpadding_tokens_per_second": 224262.5607826986, "ranks": [{"compute_seconds": 0.3771145199999637, "nonpadding_tokens": 10288, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3809681320000209, "nonpadding_tokens": 14419, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3774476920000325, "nonpadding_tokens": 13563, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3810449750000089, "nonpadding_tokens": 11404, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3775497289999521, "nonpadding_tokens": 1417, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3768648180000582, "nonpadding_tokens": 6670, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.377869239000006, "nonpadding_tokens": 18230, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3788592590000235, "nonpadding_tokens": 9995, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00041722700007085223, "prefetch": {"data_wait_seconds": 0.0018564360009349912, "consumed_batches": 190, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 195, "tokens": 17420707, "loss_tokens": 107793, "loss": 5.51361289694136, "lr": 0.00026131061999999997, "optimizer_lrs": [0.017420708, 0.00026131061999999997], "gradient_norm": 1.0492264032363892, "seconds": 0.40129295499991713, "nonpadding_tokens_per_second": 268614.2347054716, "ranks": [{"compute_seconds": 0.3987585370000488, "nonpadding_tokens": 13007, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39420119299995804, "nonpadding_tokens": 11601, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.39365132699992955, "nonpadding_tokens": 4273, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.397101570000018, "nonpadding_tokens": 20267, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3942779110000174, "nonpadding_tokens": 11895, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.394300674999954, "nonpadding_tokens": 17035, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.39722316800009594, "nonpadding_tokens": 13072, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3968143230000578, "nonpadding_tokens": 16643, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003425649999826419, "prefetch": {"data_wait_seconds": 0.0018658820008568, "consumed_batches": 191, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 196, "tokens": 17585661, "loss_tokens": 164954, "loss": 5.421941116917444, "lr": 0.00026378492999999995, "optimizer_lrs": [0.017585662, 0.00026378492999999995], "gradient_norm": 1.2900604009628296, "seconds": 0.4473278080000682, "nonpadding_tokens_per_second": 368754.18216784514, "ranks": [{"compute_seconds": 0.4444688409998889, "nonpadding_tokens": 20320, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.44495829400000275, "nonpadding_tokens": 30786, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4404396549999774, "nonpadding_tokens": 32401, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.43990418700002465, "nonpadding_tokens": 25183, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.44069784500004516, "nonpadding_tokens": 20228, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4404186529999379, "nonpadding_tokens": 4824, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.44318788099997164, "nonpadding_tokens": 26381, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.44296964800003025, "nonpadding_tokens": 4831, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040939400003026094, "prefetch": {"data_wait_seconds": 0.0018751410008235325, "consumed_batches": 192, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 197, "tokens": 17669506, "loss_tokens": 83845, "loss": 5.332416214443318, "lr": 0.00026504260499999997, "optimizer_lrs": [0.017669506999999997, 0.00026504260499999997], "gradient_norm": 0.8580498099327087, "seconds": 0.3498816329999954, "nonpadding_tokens_per_second": 239638.18643775766, "ranks": [{"compute_seconds": 0.3442822660000502, "nonpadding_tokens": 10820, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3475681559999657, "nonpadding_tokens": 18681, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34364928799993777, "nonpadding_tokens": 9893, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3428088740000703, "nonpadding_tokens": 3782, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34634372799996527, "nonpadding_tokens": 9570, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34312578300000496, "nonpadding_tokens": 4817, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3460887979999825, "nonpadding_tokens": 10901, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3465991919999851, "nonpadding_tokens": 15381, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003491119999807779, "prefetch": {"data_wait_seconds": 0.0018843380007638189, "consumed_batches": 193, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 198, "tokens": 17774958, "loss_tokens": 105452, "loss": 5.573147616925236, "lr": 0.00026662438499999995, "optimizer_lrs": [0.017774958999999996, 0.00026662438499999995], "gradient_norm": 0.8888470530509949, "seconds": 0.362916764999909, "nonpadding_tokens_per_second": 290568.0039334266, "ranks": [{"compute_seconds": 0.3601992410000321, "nonpadding_tokens": 20348, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3570095200000196, "nonpadding_tokens": 17171, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35704610699997374, "nonpadding_tokens": 6812, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.356056738999996, "nonpadding_tokens": 9632, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35952261400007046, "nonpadding_tokens": 11421, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35667979200002264, "nonpadding_tokens": 16804, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.35813315200005036, "nonpadding_tokens": 10247, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35942535899994255, "nonpadding_tokens": 13017, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003686620000280527, "prefetch": {"data_wait_seconds": 0.001893218000759589, "consumed_batches": 194, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 199, "tokens": 17863968, "loss_tokens": 89010, "loss": 5.441532903606336, "lr": 0.000267959535, "optimizer_lrs": [0.017863969, 0.000267959535], "gradient_norm": 1.0922887325286865, "seconds": 0.35350618500001474, "nonpadding_tokens_per_second": 251791.91702118673, "ranks": [{"compute_seconds": 0.3485817550000547, "nonpadding_tokens": 16575, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3476318020000235, "nonpadding_tokens": 24312, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34799132399996324, "nonpadding_tokens": 3277, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35058716700007153, "nonpadding_tokens": 16846, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3508788660000164, "nonpadding_tokens": 4276, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34768590299995594, "nonpadding_tokens": 6274, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.34906972699991456, "nonpadding_tokens": 8200, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35057330999995884, "nonpadding_tokens": 9250, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003909379998958684, "prefetch": {"data_wait_seconds": 0.0019023860007791882, "consumed_batches": 195, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 200, "tokens": 17967165, "loss_tokens": 103197, "loss": 5.459904357684817, "lr": 0.00026950749, "optimizer_lrs": [0.017967166, 0.00026950749], "gradient_norm": 0.7206336855888367, "seconds": 0.38754708399994797, "nonpadding_tokens_per_second": 266282.4834982215, "ranks": [{"compute_seconds": 0.38380772099992555, "nonpadding_tokens": 13345, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3812690230000726, "nonpadding_tokens": 4433, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3812192130000085, "nonpadding_tokens": 11035, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3850378949999822, "nonpadding_tokens": 15097, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.38359127599994736, "nonpadding_tokens": 15644, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.38108045100000254, "nonpadding_tokens": 12131, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.38238945099999455, "nonpadding_tokens": 22825, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3814871590000166, "nonpadding_tokens": 8687, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000592932000017754, "prefetch": {"data_wait_seconds": 0.0019121610007459822, "consumed_batches": 196, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 201, "tokens": 18056466, "loss_tokens": 89301, "loss": 5.085003247444037, "lr": 0.000270847005, "optimizer_lrs": [0.018056467, 0.000270847005], "gradient_norm": 0.754613995552063, "seconds": 0.33486719500001527, "nonpadding_tokens_per_second": 266675.86832444405, "ranks": [{"compute_seconds": 0.3290698460000385, "nonpadding_tokens": 9105, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33171524300007604, "nonpadding_tokens": 10251, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3288579649999974, "nonpadding_tokens": 6363, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33252754299996923, "nonpadding_tokens": 7661, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33098634800001037, "nonpadding_tokens": 23196, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3288603770000691, "nonpadding_tokens": 15477, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3316564270000981, "nonpadding_tokens": 8228, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32938174399998843, "nonpadding_tokens": 9020, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004123730000173964, "prefetch": {"data_wait_seconds": 0.0019224780007789377, "consumed_batches": 197, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 202, "tokens": 18139683, "loss_tokens": 83217, "loss": 5.3872725074203585, "lr": 0.00027209525999999997, "optimizer_lrs": [0.018139684, 0.00027209525999999997], "gradient_norm": 0.7117672562599182, "seconds": 0.3501852050000025, "nonpadding_tokens_per_second": 237637.10976881336, "ranks": [{"compute_seconds": 0.34629736099998354, "nonpadding_tokens": 12012, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3477203710000367, "nonpadding_tokens": 13920, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3465372140000227, "nonpadding_tokens": 5798, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3478815169999052, "nonpadding_tokens": 11324, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3464692829999194, "nonpadding_tokens": 16370, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34405601200001, "nonpadding_tokens": 3281, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.34697095100000297, "nonpadding_tokens": 9030, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34379166700000496, "nonpadding_tokens": 11482, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003701690000070812, "prefetch": {"data_wait_seconds": 0.0019317800007456754, "consumed_batches": 198, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 203, "tokens": 18244183, "loss_tokens": 104500, "loss": 5.447669258373206, "lr": 0.00027366276, "optimizer_lrs": [0.018244184, 0.00027366276], "gradient_norm": 0.7247090339660645, "seconds": 0.2849291530000073, "nonpadding_tokens_per_second": 366757.8375175857, "ranks": [{"compute_seconds": 0.28051635400004216, "nonpadding_tokens": 16986, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2826456439998992, "nonpadding_tokens": 6269, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2792874170000914, "nonpadding_tokens": 14028, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2779775819999486, "nonpadding_tokens": 11695, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2813257920000751, "nonpadding_tokens": 11867, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2785324749999063, "nonpadding_tokens": 8349, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.28129583800000546, "nonpadding_tokens": 18062, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.27918671300005826, "nonpadding_tokens": 17244, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003545349999285463, "prefetch": {"data_wait_seconds": 0.0019400210006779162, "consumed_batches": 199, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 204, "tokens": 18341644, "loss_tokens": 97461, "loss": 5.093709599224305, "lr": 0.000275124675, "optimizer_lrs": [0.018341645, 0.000275124675], "gradient_norm": 0.7832214832305908, "seconds": 0.3729051560000016, "nonpadding_tokens_per_second": 261356.0001299622, "ranks": [{"compute_seconds": 0.36726218799992694, "nonpadding_tokens": 6404, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.37064140400002543, "nonpadding_tokens": 12663, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.368491382000002, "nonpadding_tokens": 15963, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36499346899995544, "nonpadding_tokens": 18498, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36864668800001255, "nonpadding_tokens": 1016, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3689926810000088, "nonpadding_tokens": 9927, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3698765080000612, "nonpadding_tokens": 16918, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3650673289999986, "nonpadding_tokens": 16072, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003397800001039286, "prefetch": {"data_wait_seconds": 0.001949592000642042, "consumed_batches": 200, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 205, "tokens": 18425201, "loss_tokens": 83557, "loss": 4.775314306401618, "lr": 0.00027637802999999996, "optimizer_lrs": [0.018425201999999998, 0.00027637802999999996], "gradient_norm": 1.083998680114746, "seconds": 0.3608992459999172, "nonpadding_tokens_per_second": 231524.4515640223, "ranks": [{"compute_seconds": 0.3557732689999966, "nonpadding_tokens": 5968, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3571450649999406, "nonpadding_tokens": 18014, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35635768199995255, "nonpadding_tokens": 14209, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3576473310000665, "nonpadding_tokens": 15767, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35383806599998024, "nonpadding_tokens": 4910, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3585345569999845, "nonpadding_tokens": 4970, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.356794079999986, "nonpadding_tokens": 14642, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3568319089999932, "nonpadding_tokens": 5077, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037296999994396174, "prefetch": {"data_wait_seconds": 0.0019579870006509736, "consumed_batches": 201, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 206, "tokens": 18494873, "loss_tokens": 69672, "loss": 5.387076228614077, "lr": 0.00027742310999999994, "optimizer_lrs": [0.018494873999999998, 0.00027742310999999994], "gradient_norm": 0.982094943523407, "seconds": 0.34376076600005945, "nonpadding_tokens_per_second": 202675.83415842152, "ranks": [{"compute_seconds": 0.34022499000002426, "nonpadding_tokens": 11306, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3413092860000688, "nonpadding_tokens": 2701, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33753919099990526, "nonpadding_tokens": 9521, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3413830859999507, "nonpadding_tokens": 3975, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3399786889999632, "nonpadding_tokens": 10762, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33653937700000824, "nonpadding_tokens": 10631, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33959847300002366, "nonpadding_tokens": 11742, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3392423070000632, "nonpadding_tokens": 9034, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003862830000116446, "prefetch": {"data_wait_seconds": 0.0019673740007419838, "consumed_batches": 202, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 207, "tokens": 18559377, "loss_tokens": 64504, "loss": 5.346915016898177, "lr": 0.00027839067, "optimizer_lrs": [0.018559378, 0.00027839067], "gradient_norm": 0.9551976323127747, "seconds": 0.46739288300000226, "nonpadding_tokens_per_second": 138008.09200597025, "ranks": [{"compute_seconds": 0.46420948000002227, "nonpadding_tokens": 4398, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.465055561999975, "nonpadding_tokens": 3058, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.463488032999976, "nonpadding_tokens": 7380, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4613896339999428, "nonpadding_tokens": 14427, "peak_allocated_bytes": 14989287936, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4644034149999925, "nonpadding_tokens": 7564, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.461432076000051, "nonpadding_tokens": 12218, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.46430992799992055, "nonpadding_tokens": 7286, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4620054210000717, "nonpadding_tokens": 8173, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040381099995556724, "prefetch": {"data_wait_seconds": 0.0019774480007299644, "consumed_batches": 203, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 208, "tokens": 18678431, "loss_tokens": 119054, "loss": 5.7573097081996405, "lr": 0.00028017647999999996, "optimizer_lrs": [0.018678432, 0.00028017647999999996], "gradient_norm": 0.9702599048614502, "seconds": 0.31310885400000643, "nonpadding_tokens_per_second": 380231.9815587123, "ranks": [{"compute_seconds": 0.31016255500003354, "nonpadding_tokens": 13419, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3105550460001041, "nonpadding_tokens": 14796, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3067241070000364, "nonpadding_tokens": 19242, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31030899899997166, "nonpadding_tokens": 18010, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3092609599999605, "nonpadding_tokens": 10598, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3107056049999528, "nonpadding_tokens": 18066, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.30699827799992363, "nonpadding_tokens": 10853, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3093988589999981, "nonpadding_tokens": 14070, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003879270000197721, "prefetch": {"data_wait_seconds": 0.001987460000691499, "consumed_batches": 204, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 209, "tokens": 18793118, "loss_tokens": 114687, "loss": 5.322728382467062, "lr": 0.00028189678499999997, "optimizer_lrs": [0.018793119, 0.00028189678499999997], "gradient_norm": 1.2534050941467285, "seconds": 0.29285410900001807, "nonpadding_tokens_per_second": 391618.2033149923, "ranks": [{"compute_seconds": 0.28886995399989246, "nonpadding_tokens": 4632, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.28618681599994034, "nonpadding_tokens": 7616, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.28672556799995164, "nonpadding_tokens": 19833, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.290496834999999, "nonpadding_tokens": 18689, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.28899676900005034, "nonpadding_tokens": 16233, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.28613399500000014, "nonpadding_tokens": 18221, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2871631210000487, "nonpadding_tokens": 10269, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.28904311899998447, "nonpadding_tokens": 19194, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039168899991182116, "prefetch": {"data_wait_seconds": 0.0019974310006318774, "consumed_batches": 205, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 210, "tokens": 18891717, "loss_tokens": 98599, "loss": 5.303960993519204, "lr": 0.00028337576999999995, "optimizer_lrs": [0.018891718, 0.00028337576999999995], "gradient_norm": 1.0054701566696167, "seconds": 0.37112588399998003, "nonpadding_tokens_per_second": 265675.35235565866, "ranks": [{"compute_seconds": 0.3666794949999712, "nonpadding_tokens": 16393, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.368808796000053, "nonpadding_tokens": 18768, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3676369979999663, "nonpadding_tokens": 17852, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3645630580000443, "nonpadding_tokens": 10103, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3657363749999831, "nonpadding_tokens": 938, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36501657899998463, "nonpadding_tokens": 7761, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3658791770000107, "nonpadding_tokens": 14223, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3681569180000679, "nonpadding_tokens": 12561, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003697229999488627, "prefetch": {"data_wait_seconds": 0.002007390000585474, "consumed_batches": 206, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 211, "tokens": 18968005, "loss_tokens": 76288, "loss": 5.348921193372483, "lr": 0.00028452009, "optimizer_lrs": [0.018968006, 0.00028452009], "gradient_norm": 0.9163548946380615, "seconds": 0.38707236599998396, "nonpadding_tokens_per_second": 197089.76072965944, "ranks": [{"compute_seconds": 0.33167731000003187, "nonpadding_tokens": 16684, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34133446700002423, "nonpadding_tokens": 7642, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33766494999997576, "nonpadding_tokens": 11843, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34149658100000124, "nonpadding_tokens": 6703, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33753145600007883, "nonpadding_tokens": 3886, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.336977395999952, "nonpadding_tokens": 4299, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33775399000001016, "nonpadding_tokens": 14788, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3381501579999622, "nonpadding_tokens": 10443, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.014462324000078297, "prefetch": {"data_wait_seconds": 0.0020155810005917374, "consumed_batches": 207, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 212, "tokens": 19099302, "loss_tokens": 131297, "loss": 4.968558877963701, "lr": 0.000286489545, "optimizer_lrs": [0.019099303, 0.000286489545], "gradient_norm": 1.044951319694519, "seconds": 0.37466099400000985, "nonpadding_tokens_per_second": 350442.13863372325, "ranks": [{"compute_seconds": 0.36955747499996505, "nonpadding_tokens": 9955, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.37036161800006084, "nonpadding_tokens": 16316, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3695161340000368, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3704050520000237, "nonpadding_tokens": 10323, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36716514600004757, "nonpadding_tokens": 16532, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37211438700001054, "nonpadding_tokens": 17233, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3694177200000013, "nonpadding_tokens": 15959, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3691661299999396, "nonpadding_tokens": 12211, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039794499991785415, "prefetch": {"data_wait_seconds": 0.0020253310005955427, "consumed_batches": 208, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 213, "tokens": 19184804, "loss_tokens": 85502, "loss": 5.547796396575519, "lr": 0.00028777207499999997, "optimizer_lrs": [0.019184805, 0.00028777207499999997], "gradient_norm": 0.9570707678794861, "seconds": 0.3179055850000623, "nonpadding_tokens_per_second": 268954.06697552436, "ranks": [{"compute_seconds": 0.29478819200005546, "nonpadding_tokens": 6604, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30737596599999506, "nonpadding_tokens": 25326, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3041276180000523, "nonpadding_tokens": 2859, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3074217570000428, "nonpadding_tokens": 7732, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3037720500000205, "nonpadding_tokens": 17277, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30784541700006685, "nonpadding_tokens": 13549, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3065209420000201, "nonpadding_tokens": 7078, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3040559519999988, "nonpadding_tokens": 5077, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004881039999418135, "prefetch": {"data_wait_seconds": 0.002035533000594114, "consumed_batches": 209, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 214, "tokens": 19271005, "loss_tokens": 86201, "loss": 5.2931161616454565, "lr": 0.00028906509, "optimizer_lrs": [0.019271006, 0.00028906509], "gradient_norm": 0.8296104073524475, "seconds": 0.3284414359999346, "nonpadding_tokens_per_second": 262454.704405863, "ranks": [{"compute_seconds": 0.32337032800001, "nonpadding_tokens": 3558, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3260902319999559, "nonpadding_tokens": 9051, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32248323599992545, "nonpadding_tokens": 9421, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3216319379999959, "nonpadding_tokens": 12385, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32225840199998856, "nonpadding_tokens": 7123, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32298702700006743, "nonpadding_tokens": 16529, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32314230399992994, "nonpadding_tokens": 7088, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32256650500005435, "nonpadding_tokens": 21046, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000393938000001981, "prefetch": {"data_wait_seconds": 0.0020458220005821204, "consumed_batches": 210, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 215, "tokens": 19380973, "loss_tokens": 109968, "loss": 4.9909513449367084, "lr": 0.00029071460999999997, "optimizer_lrs": [0.019380974, 0.00029071460999999997], "gradient_norm": 1.1042468547821045, "seconds": 0.3211478900000202, "nonpadding_tokens_per_second": 342421.67993067956, "ranks": [{"compute_seconds": 0.3187614849999818, "nonpadding_tokens": 16472, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31783987699998306, "nonpadding_tokens": 6667, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3175931189999801, "nonpadding_tokens": 8557, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3186884659999123, "nonpadding_tokens": 17666, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3177421919999688, "nonpadding_tokens": 12390, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31466885800000455, "nonpadding_tokens": 13018, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3156330199999502, "nonpadding_tokens": 15174, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3177407419999554, "nonpadding_tokens": 20024, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003840950000721932, "prefetch": {"data_wait_seconds": 0.0020548220005593976, "consumed_batches": 211, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 216, "tokens": 19492228, "loss_tokens": 111255, "loss": 5.70946699024763, "lr": 0.000292383435, "optimizer_lrs": [0.019492229000000003, 0.000292383435], "gradient_norm": 1.6877496242523193, "seconds": 0.31813601500005007, "nonpadding_tokens_per_second": 349708.91302571475, "ranks": [{"compute_seconds": 0.3154790950000006, "nonpadding_tokens": 8369, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31225780299996586, "nonpadding_tokens": 12654, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3129927029999635, "nonpadding_tokens": 19865, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3121345819999988, "nonpadding_tokens": 10101, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3153615549999813, "nonpadding_tokens": 11947, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3125793929999645, "nonpadding_tokens": 15458, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31577336999998806, "nonpadding_tokens": 12921, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3134288609999203, "nonpadding_tokens": 19940, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003870319999350613, "prefetch": {"data_wait_seconds": 0.0020652610005527094, "consumed_batches": 212, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 217, "tokens": 19564111, "loss_tokens": 71883, "loss": 5.459346698802221, "lr": 0.00029346167999999995, "optimizer_lrs": [0.019564111999999998, 0.00029346167999999995], "gradient_norm": 1.0228177309036255, "seconds": 0.2842536690000088, "nonpadding_tokens_per_second": 252883.27940631707, "ranks": [{"compute_seconds": 0.27921582700003, "nonpadding_tokens": 3045, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.28190722400006507, "nonpadding_tokens": 7803, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.28054999599999064, "nonpadding_tokens": 14425, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2770953600000894, "nonpadding_tokens": 10944, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.280453702999921, "nonpadding_tokens": 7918, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.27797206100001404, "nonpadding_tokens": 13068, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2810021369999731, "nonpadding_tokens": 6314, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2807630720000134, "nonpadding_tokens": 8366, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038137100000312785, "prefetch": {"data_wait_seconds": 0.0020761690005883793, "consumed_batches": 213, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 218, "tokens": 19648738, "loss_tokens": 84627, "loss": 5.454245911470334, "lr": 0.000294731085, "optimizer_lrs": [0.019648739000000002, 0.000294731085], "gradient_norm": 0.9259387850761414, "seconds": 0.35541474199999357, "nonpadding_tokens_per_second": 238107.73724180955, "ranks": [{"compute_seconds": 0.3523770680000098, "nonpadding_tokens": 7738, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3530654679999543, "nonpadding_tokens": 12084, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34935474200005956, "nonpadding_tokens": 14484, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3491060329999982, "nonpadding_tokens": 4405, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35027581299993926, "nonpadding_tokens": 6234, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3484280719999333, "nonpadding_tokens": 13812, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.35026200300001165, "nonpadding_tokens": 12685, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3504096719999552, "nonpadding_tokens": 13185, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003678090000676093, "prefetch": {"data_wait_seconds": 0.0020864480005684527, "consumed_batches": 214, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 219, "tokens": 19711698, "loss_tokens": 62960, "loss": 5.103870513024142, "lr": 0.000295675485, "optimizer_lrs": [0.019711699, 0.000295675485], "gradient_norm": 0.8496477007865906, "seconds": 0.36742910199996004, "nonpadding_tokens_per_second": 171352.78522387389, "ranks": [{"compute_seconds": 0.3629642910000257, "nonpadding_tokens": 8430, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.36515887199993813, "nonpadding_tokens": 7908, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3622662399999399, "nonpadding_tokens": 10761, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.364786691000063, "nonpadding_tokens": 8817, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.359376768000061, "nonpadding_tokens": 5386, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3635928559999684, "nonpadding_tokens": 9801, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.36478231700004926, "nonpadding_tokens": 5678, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36380808499995965, "nonpadding_tokens": 6179, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039804100003948406, "prefetch": {"data_wait_seconds": 0.002097297000659637, "consumed_batches": 215, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 220, "tokens": 19777807, "loss_tokens": 66109, "loss": 5.071565047875478, "lr": 0.00029666712, "optimizer_lrs": [0.019777808, 0.00029666712], "gradient_norm": 0.8520146012306213, "seconds": 0.3178168700000015, "nonpadding_tokens_per_second": 208009.72585250018, "ranks": [{"compute_seconds": 0.3150555549999581, "nonpadding_tokens": 2502, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31548035100001925, "nonpadding_tokens": 6411, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3119385569999622, "nonpadding_tokens": 7314, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31543911799997204, "nonpadding_tokens": 3960, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3141911100000243, "nonpadding_tokens": 10717, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31182715800002825, "nonpadding_tokens": 11103, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31198012099991956, "nonpadding_tokens": 16020, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3142328810000663, "nonpadding_tokens": 8082, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036557900000389054, "prefetch": {"data_wait_seconds": 0.0021063440005946177, "consumed_batches": 216, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 221, "tokens": 19850457, "loss_tokens": 72650, "loss": 5.479883860977289, "lr": 0.00029775687, "optimizer_lrs": [0.019850458, 0.00029775687], "gradient_norm": 0.7456226944923401, "seconds": 0.3332637570000543, "nonpadding_tokens_per_second": 217995.50198309793, "ranks": [{"compute_seconds": 0.32253109600003427, "nonpadding_tokens": 6181, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3308585759999687, "nonpadding_tokens": 12084, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3213911120000148, "nonpadding_tokens": 4833, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32488721200002146, "nonpadding_tokens": 17005, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32153644900006384, "nonpadding_tokens": 6372, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3258393490000344, "nonpadding_tokens": 3892, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3241772609999316, "nonpadding_tokens": 10201, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32143730899997536, "nonpadding_tokens": 12082, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003666549999934432, "prefetch": {"data_wait_seconds": 0.0021172260005641874, "consumed_batches": 217, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 222, "tokens": 19952730, "loss_tokens": 102273, "loss": 5.680785129017433, "lr": 0.00029929096499999995, "optimizer_lrs": [0.019952730999999998, 0.00029929096499999995], "gradient_norm": 0.954242467880249, "seconds": 0.40430160599999, "nonpadding_tokens_per_second": 252962.14133762935, "ranks": [{"compute_seconds": 0.39833524699997724, "nonpadding_tokens": 6551, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40192580100006126, "nonpadding_tokens": 10785, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4002603349999845, "nonpadding_tokens": 9150, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3968749480000042, "nonpadding_tokens": 10008, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3978548420000152, "nonpadding_tokens": 20431, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.397825115000046, "nonpadding_tokens": 20088, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3980606769999895, "nonpadding_tokens": 13193, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4006010329999299, "nonpadding_tokens": 12067, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003523750000340442, "prefetch": {"data_wait_seconds": 0.0021259930005044225, "consumed_batches": 218, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 223, "tokens": 20024709, "loss_tokens": 71979, "loss": 5.100860146709457, "lr": 0.00029999999998359957, "optimizer_lrs": [0.01999999999890664, 0.00029999999998359957], "gradient_norm": 1.281935214996338, "seconds": 0.37470093800004634, "nonpadding_tokens_per_second": 192097.19725866045, "ranks": [{"compute_seconds": 0.3676296790000606, "nonpadding_tokens": 8565, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3709389520000741, "nonpadding_tokens": 2475, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3673637450000342, "nonpadding_tokens": 19609, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37092973500000426, "nonpadding_tokens": 10377, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3659749240000565, "nonpadding_tokens": 5027, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3670381829999769, "nonpadding_tokens": 7024, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3693631980000873, "nonpadding_tokens": 5105, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3672968009999522, "nonpadding_tokens": 13797, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000348640000083833, "prefetch": {"data_wait_seconds": 0.0021351540004843628, "consumed_batches": 219, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 224, "tokens": 20141441, "loss_tokens": 116732, "loss": 5.395135331357297, "lr": 0.0002999999994626027, "optimizer_lrs": [0.019999999964173516, 0.0002999999994626027], "gradient_norm": 3.9154646396636963, "seconds": 0.3569174109999267, "nonpadding_tokens_per_second": 327056.05387242924, "ranks": [{"compute_seconds": 0.35164798999994673, "nonpadding_tokens": 12409, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34922977799999444, "nonpadding_tokens": 21830, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3492148489999636, "nonpadding_tokens": 14364, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3531228120000378, "nonpadding_tokens": 11058, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35002660199995717, "nonpadding_tokens": 18176, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3544200329999967, "nonpadding_tokens": 5378, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.35258549699995, "nonpadding_tokens": 25798, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34919092100005855, "nonpadding_tokens": 7719, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00045132799994007655, "prefetch": {"data_wait_seconds": 0.0021460100005015192, "consumed_batches": 220, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 225, "tokens": 20234602, "loss_tokens": 93161, "loss": 4.914279110894044, "lr": 0.0002999999985215445, "optimizer_lrs": [0.0199999999014363, 0.0002999999985215445], "gradient_norm": 0.8000162839889526, "seconds": 0.6005581100000654, "nonpadding_tokens_per_second": 155124.03953713962, "ranks": [{"compute_seconds": 0.5965840309999066, "nonpadding_tokens": 6516, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.5928710919999958, "nonpadding_tokens": 8655, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.5938318710000203, "nonpadding_tokens": 17091, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.597582310000007, "nonpadding_tokens": 4880, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.5938917810000248, "nonpadding_tokens": 9947, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.5979771889999483, "nonpadding_tokens": 11807, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.596346330000074, "nonpadding_tokens": 12817, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.593884045999971, "nonpadding_tokens": 21448, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00041614500003106514, "prefetch": {"data_wait_seconds": 0.0021714930005600763, "consumed_batches": 221, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 226, "tokens": 20356756, "loss_tokens": 122154, "loss": 5.0762464798533, "lr": 0.0002999999965810921, "optimizer_lrs": [0.019999999772072806, 0.0002999999965810921], "gradient_norm": 1.023327112197876, "seconds": 0.3002045010000529, "nonpadding_tokens_per_second": 406902.6266863949, "ranks": [{"compute_seconds": 0.29667383299999983, "nonpadding_tokens": 10905, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2975801520000232, "nonpadding_tokens": 18022, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2962698219999993, "nonpadding_tokens": 25444, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2971404809999285, "nonpadding_tokens": 13704, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2964519779999364, "nonpadding_tokens": 14175, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2941267509999079, "nonpadding_tokens": 7264, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2967557219999435, "nonpadding_tokens": 10911, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.29665769799999, "nonpadding_tokens": 21729, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003811070000665495, "prefetch": {"data_wait_seconds": 0.002181682000582441, "consumed_batches": 222, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 227, "tokens": 20451013, "loss_tokens": 94257, "loss": 5.294842491804323, "lr": 0.0002999999945358457, "optimizer_lrs": [0.019999999635723048, 0.0002999999945358457], "gradient_norm": 0.7818959355354309, "seconds": 0.3188152440000067, "nonpadding_tokens_per_second": 295647.7200318502, "ranks": [{"compute_seconds": 0.31415201899994827, "nonpadding_tokens": 9518, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3117583389999936, "nonpadding_tokens": 3824, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3128815960000111, "nonpadding_tokens": 6992, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3109871930000736, "nonpadding_tokens": 21719, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31503129300006094, "nonpadding_tokens": 18916, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31645355000000563, "nonpadding_tokens": 13336, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31601162399999794, "nonpadding_tokens": 8359, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31408626400002504, "nonpadding_tokens": 11593, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003511529999968843, "prefetch": {"data_wait_seconds": 0.0021907650005914547, "consumed_batches": 223, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 228, "tokens": 20575968, "loss_tokens": 124955, "loss": 4.886248049297747, "lr": 0.00029999999108869024, "optimizer_lrs": [0.019999999405912683, 0.00029999999108869024], "gradient_norm": 1.9482933282852173, "seconds": 0.34349209900005917, "nonpadding_tokens_per_second": 363778.3819882811, "ranks": [{"compute_seconds": 0.33959290300003886, "nonpadding_tokens": 21385, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3399854139998979, "nonpadding_tokens": 11559, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33628324500000417, "nonpadding_tokens": 10165, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3393470849999858, "nonpadding_tokens": 17954, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3372706369999605, "nonpadding_tokens": 15134, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34099609700001565, "nonpadding_tokens": 21383, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33673184400004175, "nonpadding_tokens": 12008, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33937687099989944, "nonpadding_tokens": 15367, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038710800004082557, "prefetch": {"data_wait_seconds": 0.0022019710005452, "consumed_batches": 224, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 229, "tokens": 20707929, "loss_tokens": 131961, "loss": 5.073713540364198, "lr": 0.000299999986537546, "optimizer_lrs": [0.01999999910250307, 0.000299999986537546], "gradient_norm": 1.4055105447769165, "seconds": 0.3581914769999912, "nonpadding_tokens_per_second": 368409.10092342383, "ranks": [{"compute_seconds": 0.35520102999998926, "nonpadding_tokens": 16656, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35580526199998985, "nonpadding_tokens": 15607, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35439724200000455, "nonpadding_tokens": 12789, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35490878500002054, "nonpadding_tokens": 19719, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3521760290000202, "nonpadding_tokens": 3703, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3512834030000249, "nonpadding_tokens": 12344, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.35496785199995884, "nonpadding_tokens": 27824, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.354445116000079, "nonpadding_tokens": 23319, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004037560000824669, "prefetch": {"data_wait_seconds": 0.0022122710005305635, "consumed_batches": 225, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 230, "tokens": 20836318, "loss_tokens": 128389, "loss": 5.220440906152397, "lr": 0.00029999998121169025, "optimizer_lrs": [0.01999999874744602, 0.00029999998121169025], "gradient_norm": 0.8019524812698364, "seconds": 0.3471690830000398, "nonpadding_tokens_per_second": 369816.91713598033, "ranks": [{"compute_seconds": 0.3422992810000096, "nonpadding_tokens": 16901, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3422945229999641, "nonpadding_tokens": 26184, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3411932669999942, "nonpadding_tokens": 13150, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3449727109999685, "nonpadding_tokens": 17062, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3413615580000169, "nonpadding_tokens": 10827, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3422973850000517, "nonpadding_tokens": 12182, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33980765299997984, "nonpadding_tokens": 11657, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33917267100002846, "nonpadding_tokens": 20426, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039010199998301687, "prefetch": {"data_wait_seconds": 0.0022229330005529846, "consumed_batches": 226, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 231, "tokens": 20903338, "loss_tokens": 67020, "loss": 5.162958818263205, "lr": 0.00029999997807975614, "optimizer_lrs": [0.01999999853865041, 0.00029999997807975614], "gradient_norm": 0.9185174703598022, "seconds": 0.31119752699999026, "nonpadding_tokens_per_second": 215361.60857731398, "ranks": [{"compute_seconds": 0.3071648919999461, "nonpadding_tokens": 6809, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.308892893999996, "nonpadding_tokens": 4606, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3074155259999998, "nonpadding_tokens": 1834, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30461857399996006, "nonpadding_tokens": 9101, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3052791690000731, "nonpadding_tokens": 4724, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3045337629999949, "nonpadding_tokens": 10941, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3078947999999855, "nonpadding_tokens": 17628, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3047913599999674, "nonpadding_tokens": 11377, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003646100000196384, "prefetch": {"data_wait_seconds": 0.002232151000498561, "consumed_batches": 227, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 232, "tokens": 20978350, "loss_tokens": 75012, "loss": 5.039521593211753, "lr": 0.00029999997428815044, "optimizer_lrs": [0.0199999982858767, 0.00029999997428815044], "gradient_norm": 0.7476861476898193, "seconds": 0.29833997099990484, "nonpadding_tokens_per_second": 251431.27737323515, "ranks": [{"compute_seconds": 0.2940384380000296, "nonpadding_tokens": 4499, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2959813600000416, "nonpadding_tokens": 4837, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2924365169999419, "nonpadding_tokens": 7585, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2958115620000399, "nonpadding_tokens": 10598, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2911612770000147, "nonpadding_tokens": 12183, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.29124006799997915, "nonpadding_tokens": 4496, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.29240763799998604, "nonpadding_tokens": 10328, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2947975329999508, "nonpadding_tokens": 20486, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038260899998476816, "prefetch": {"data_wait_seconds": 0.0022416190005287717, "consumed_batches": 228, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 233, "tokens": 21081749, "loss_tokens": 103399, "loss": 5.223593796845231, "lr": 0.0002999999685661323, "optimizer_lrs": [0.01999999790440882, 0.0002999999685661323], "gradient_norm": 0.9684494733810425, "seconds": 0.33170928000004096, "nonpadding_tokens_per_second": 311715.7289057069, "ranks": [{"compute_seconds": 0.3291590470000756, "nonpadding_tokens": 14521, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3293424099999811, "nonpadding_tokens": 11131, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3254712740000514, "nonpadding_tokens": 17405, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32484937499998523, "nonpadding_tokens": 8402, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32704286900002444, "nonpadding_tokens": 14679, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32530672000007144, "nonpadding_tokens": 17220, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3278849610001089, "nonpadding_tokens": 9960, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3281981599999426, "nonpadding_tokens": 10081, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003878490000488455, "prefetch": {"data_wait_seconds": 0.0022513630005960295, "consumed_batches": 229, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 234, "tokens": 21140430, "loss_tokens": 58681, "loss": 4.976143257613197, "lr": 0.0002999999650632839, "optimizer_lrs": [0.019999997670885596, 0.0002999999650632839], "gradient_norm": 0.6581354737281799, "seconds": 0.31604367599993566, "nonpadding_tokens_per_second": 185673.7041623701, "ranks": [{"compute_seconds": 0.31242255999995905, "nonpadding_tokens": 7518, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31352111599994714, "nonpadding_tokens": 5657, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3099919039999577, "nonpadding_tokens": 5079, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30882470500000636, "nonpadding_tokens": 11402, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.312511144000041, "nonpadding_tokens": 9162, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3098173299999871, "nonpadding_tokens": 4822, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3095813280000357, "nonpadding_tokens": 7662, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30939702199998464, "nonpadding_tokens": 7379, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003637710000248262, "prefetch": {"data_wait_seconds": 0.0022603630005733066, "consumed_batches": 230, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 235, "tokens": 21205027, "loss_tokens": 64597, "loss": 5.021507384243851, "lr": 0.00029999996099337565, "optimizer_lrs": [0.019999997399558378, 0.00029999996099337565], "gradient_norm": 1.159522533416748, "seconds": 0.3232865859999947, "nonpadding_tokens_per_second": 199813.42498386573, "ranks": [{"compute_seconds": 0.320827035000093, "nonpadding_tokens": 8811, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32042449799996575, "nonpadding_tokens": 4605, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31946123700004136, "nonpadding_tokens": 10977, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3206298449999849, "nonpadding_tokens": 11764, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3185400419999951, "nonpadding_tokens": 5630, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31615088099999866, "nonpadding_tokens": 4895, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31746150899994063, "nonpadding_tokens": 11631, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31739780600003087, "nonpadding_tokens": 6284, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004048650000640919, "prefetch": {"data_wait_seconds": 0.0022879760005025673, "consumed_batches": 231, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 236, "tokens": 21290143, "loss_tokens": 85116, "loss": 5.272699169368861, "lr": 0.0002999999552883693, "optimizer_lrs": [0.01999999701922462, 0.0002999999552883693], "gradient_norm": 0.7608016729354858, "seconds": 0.46255970499998966, "nonpadding_tokens_per_second": 184010.84028709744, "ranks": [{"compute_seconds": 0.46005751800009875, "nonpadding_tokens": 10767, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4555665830000635, "nonpadding_tokens": 9841, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4561629519999997, "nonpadding_tokens": 18046, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.45927006300007633, "nonpadding_tokens": 12686, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.45845573899998726, "nonpadding_tokens": 3598, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.45580901699997867, "nonpadding_tokens": 12254, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4570922469999914, "nonpadding_tokens": 7697, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4586555780000481, "nonpadding_tokens": 10227, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039515900004971627, "prefetch": {"data_wait_seconds": 0.0022979350005698507, "consumed_batches": 232, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 237, "tokens": 21371309, "loss_tokens": 81166, "loss": 5.3898087715299505, "lr": 0.0002999999494855698, "optimizer_lrs": [0.019999996632371324, 0.0002999999494855698], "gradient_norm": 0.9904751777648926, "seconds": 0.38789012199993067, "nonpadding_tokens_per_second": 209249.97930216565, "ranks": [{"compute_seconds": 0.38367536199996266, "nonpadding_tokens": 9773, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3844378129999768, "nonpadding_tokens": 12940, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3818116690000579, "nonpadding_tokens": 15247, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38507185099990693, "nonpadding_tokens": 12080, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3827252690000478, "nonpadding_tokens": 9299, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.38543854299996383, "nonpadding_tokens": 1982, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3807095309999795, "nonpadding_tokens": 13190, "peak_allocated_bytes": 14988154368, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.38349135700002535, "nonpadding_tokens": 6655, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003931300000203919, "prefetch": {"data_wait_seconds": 0.0023078790005683913, "consumed_batches": 233, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 238, "tokens": 21470780, "loss_tokens": 99471, "loss": 5.4269429029566405, "lr": 0.00029999994189142397, "optimizer_lrs": [0.019999996126094932, 0.00029999994189142397], "gradient_norm": 0.8185320496559143, "seconds": 0.4076606560000755, "nonpadding_tokens_per_second": 244004.415280099, "ranks": [{"compute_seconds": 0.4016256649999832, "nonpadding_tokens": 9604, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40479085900005884, "nonpadding_tokens": 12446, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.401534641000012, "nonpadding_tokens": 27724, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.40533162199994877, "nonpadding_tokens": 13701, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4013090530000909, "nonpadding_tokens": 10723, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4013041549999343, "nonpadding_tokens": 8179, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4043116440000176, "nonpadding_tokens": 12882, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.40063274600004206, "nonpadding_tokens": 4212, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003934800000706673, "prefetch": {"data_wait_seconds": 0.0023168670005588865, "consumed_batches": 234, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 239, "tokens": 21548971, "loss_tokens": 78191, "loss": 5.14961040912637, "lr": 0.0002999999355487459, "optimizer_lrs": [0.01999999570324973, 0.0002999999355487459], "gradient_norm": 0.7973750233650208, "seconds": 0.3077783089999002, "nonpadding_tokens_per_second": 254049.7420174771, "ranks": [{"compute_seconds": 0.3039957799999229, "nonpadding_tokens": 4078, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30021795599998313, "nonpadding_tokens": 9872, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30163190300004317, "nonpadding_tokens": 9526, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.305329238000013, "nonpadding_tokens": 9267, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30408117399997536, "nonpadding_tokens": 5790, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3053129950000084, "nonpadding_tokens": 18842, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3046361460000071, "nonpadding_tokens": 3792, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30084826400002385, "nonpadding_tokens": 17024, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038700799996149726, "prefetch": {"data_wait_seconds": 0.0023260190005203185, "consumed_batches": 235, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 240, "tokens": 21648719, "loss_tokens": 99748, "loss": 5.472909857841762, "lr": 0.0002999999269806304, "optimizer_lrs": [0.019999995132042026, 0.0002999999269806304], "gradient_norm": 0.5396206378936768, "seconds": 0.3834333970000898, "nonpadding_tokens_per_second": 260144.26698459088, "ranks": [{"compute_seconds": 0.377656929000068, "nonpadding_tokens": 15668, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.38042213200003516, "nonpadding_tokens": 9223, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.37885145000007014, "nonpadding_tokens": 8350, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37568352799996774, "nonpadding_tokens": 12670, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37958868299995174, "nonpadding_tokens": 9065, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.38111485499996434, "nonpadding_tokens": 23463, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.37794086500002777, "nonpadding_tokens": 5195, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37946835500008547, "nonpadding_tokens": 16114, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003630290000273817, "prefetch": {"data_wait_seconds": 0.002335083000502891, "consumed_batches": 236, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 241, "tokens": 21741753, "loss_tokens": 93034, "loss": 5.011582459100974, "lr": 0.000299999918507447, "optimizer_lrs": [0.019999994567163133, 0.000299999918507447], "gradient_norm": 0.6557374000549316, "seconds": 0.3647269560000268, "nonpadding_tokens_per_second": 255078.4867132035, "ranks": [{"compute_seconds": 0.35997635100000025, "nonpadding_tokens": 21202, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.36194579300001806, "nonpadding_tokens": 11338, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35878533699997206, "nonpadding_tokens": 2858, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3573885409999775, "nonpadding_tokens": 15632, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3575187559999904, "nonpadding_tokens": 13440, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3594685800001116, "nonpadding_tokens": 7324, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3621983849999424, "nonpadding_tokens": 9237, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36122513199995865, "nonpadding_tokens": 12003, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038827499997751147, "prefetch": {"data_wait_seconds": 0.0023443370005225006, "consumed_batches": 237, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 242, "tokens": 21853988, "loss_tokens": 112235, "loss": 4.617637657593442, "lr": 0.000299999907666644, "optimizer_lrs": [0.019999993844442936, 0.000299999907666644], "gradient_norm": 0.999235987663269, "seconds": 0.4010963849999598, "nonpadding_tokens_per_second": 279820.5224412861, "ranks": [{"compute_seconds": 0.39601056300000437, "nonpadding_tokens": 25995, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3941697190000468, "nonpadding_tokens": 20391, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3945471010000574, "nonpadding_tokens": 6586, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3986331080000127, "nonpadding_tokens": 6212, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39707182999995894, "nonpadding_tokens": 13251, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.39464776099998744, "nonpadding_tokens": 4302, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3958864350000795, "nonpadding_tokens": 17008, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.39605510600006255, "nonpadding_tokens": 18490, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039182699993034475, "prefetch": {"data_wait_seconds": 0.0023535570004469264, "consumed_batches": 238, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 243, "tokens": 21933857, "loss_tokens": 79869, "loss": 4.727353150158384, "lr": 0.00029999989953992623, "optimizer_lrs": [0.01999999330266175, 0.00029999989953992623], "gradient_norm": 0.5676130652427673, "seconds": 0.33547415999998975, "nonpadding_tokens_per_second": 238077.94913325796, "ranks": [{"compute_seconds": 0.33312866700009636, "nonpadding_tokens": 6821, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32977428000003783, "nonpadding_tokens": 11475, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33001384399995004, "nonpadding_tokens": 13156, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32980537699995693, "nonpadding_tokens": 15582, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33076168400009465, "nonpadding_tokens": 12240, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3295592069999884, "nonpadding_tokens": 8265, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33270353499995053, "nonpadding_tokens": 5169, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3307043390000217, "nonpadding_tokens": 7161, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003591130000586418, "prefetch": {"data_wait_seconds": 0.002362713000366057, "consumed_batches": 239, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 244, "tokens": 22011184, "loss_tokens": 77327, "loss": 4.680954744138528, "lr": 0.00029999989134533295, "optimizer_lrs": [0.01999999275635553, 0.00029999989134533295], "gradient_norm": 0.8204601407051086, "seconds": 0.39986824400000387, "nonpadding_tokens_per_second": 193381.19783275225, "ranks": [{"compute_seconds": 0.3955158640000036, "nonpadding_tokens": 18776, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39742435699997714, "nonpadding_tokens": 8631, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.39368459799993616, "nonpadding_tokens": 5814, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3974216049999768, "nonpadding_tokens": 9929, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39361924199999976, "nonpadding_tokens": 3989, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3930180500000233, "nonpadding_tokens": 5293, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.39633583800002725, "nonpadding_tokens": 14109, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3959993390000136, "nonpadding_tokens": 10786, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003836790000377732, "prefetch": {"data_wait_seconds": 0.002370647000361714, "consumed_batches": 240, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 245, "tokens": 22102024, "loss_tokens": 90840, "loss": 5.233224625715544, "lr": 0.0002999998813083657, "optimizer_lrs": [0.01999999208722438, 0.0002999998813083657], "gradient_norm": 0.6669020652770996, "seconds": 0.391636043999938, "nonpadding_tokens_per_second": 231950.05003169316, "ranks": [{"compute_seconds": 0.38917144700008066, "nonpadding_tokens": 19374, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3842583929999819, "nonpadding_tokens": 15569, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3865904880000244, "nonpadding_tokens": 11778, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38832850999995117, "nonpadding_tokens": 11836, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.38692329899993183, "nonpadding_tokens": 4985, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3840196979999746, "nonpadding_tokens": 4248, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3868376290000697, "nonpadding_tokens": 10617, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3870859279999195, "nonpadding_tokens": 12433, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003839949999928649, "prefetch": {"data_wait_seconds": 0.0023802930004421796, "consumed_batches": 241, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 246, "tokens": 22194914, "loss_tokens": 92890, "loss": 5.114228522984175, "lr": 0.00029999987058644106, "optimizer_lrs": [0.019999991372429405, 0.00029999987058644106], "gradient_norm": 0.5694429278373718, "seconds": 0.3681392240000605, "nonpadding_tokens_per_second": 252323.01788082416, "ranks": [{"compute_seconds": 0.36563536200003455, "nonpadding_tokens": 7591, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.36098626099999365, "nonpadding_tokens": 13563, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36465657899998405, "nonpadding_tokens": 4165, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36560401700000966, "nonpadding_tokens": 9745, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36399521700002424, "nonpadding_tokens": 15167, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3613622939999459, "nonpadding_tokens": 12114, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.36475843300002, "nonpadding_tokens": 11937, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3625741429999607, "nonpadding_tokens": 18608, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000421109999933833, "prefetch": {"data_wait_seconds": 0.0023897430004353737, "consumed_batches": 242, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 247, "tokens": 22294537, "loss_tokens": 99623, "loss": 5.368646672957048, "lr": 0.0002999998585721669, "optimizer_lrs": [0.019999990571477794, 0.0002999998585721669], "gradient_norm": 0.5749975442886353, "seconds": 0.37465470999995887, "nonpadding_tokens_per_second": 265906.17264630395, "ranks": [{"compute_seconds": 0.3714068580000003, "nonpadding_tokens": 18920, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3723042510000596, "nonpadding_tokens": 6305, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36858893399994486, "nonpadding_tokens": 4089, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36754976199995326, "nonpadding_tokens": 11475, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36819108899999264, "nonpadding_tokens": 6084, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36786728599997787, "nonpadding_tokens": 21261, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3687493260000565, "nonpadding_tokens": 16655, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37050172099998235, "nonpadding_tokens": 14834, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000355136999928618, "prefetch": {"data_wait_seconds": 0.0023994990003757266, "consumed_batches": 243, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 248, "tokens": 22405567, "loss_tokens": 111030, "loss": 4.623425538142844, "lr": 0.0002999998445539595, "optimizer_lrs": [0.019999989636930636, 0.0002999998445539595], "gradient_norm": 0.6092274785041809, "seconds": 0.3972165349999841, "nonpadding_tokens_per_second": 279520.08594003884, "ranks": [{"compute_seconds": 0.39175541800000246, "nonpadding_tokens": 21522, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39357267499997306, "nonpadding_tokens": 28209, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3926420450000023, "nonpadding_tokens": 10624, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38926573499998085, "nonpadding_tokens": 13622, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3917879340000354, "nonpadding_tokens": 10882, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.39466028000003917, "nonpadding_tokens": 7894, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3910653249999996, "nonpadding_tokens": 5264, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3892127370000935, "nonpadding_tokens": 13013, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003945350000549297, "prefetch": {"data_wait_seconds": 0.0024092640003345878, "consumed_batches": 244, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 249, "tokens": 22494415, "loss_tokens": 88848, "loss": 5.133423445097245, "lr": 0.000299999832859321, "optimizer_lrs": [0.01999998885728807, 0.000299999832859321], "gradient_norm": 0.5494428277015686, "seconds": 0.3320940959999916, "nonpadding_tokens_per_second": 267538.63158109935, "ranks": [{"compute_seconds": 0.3280738170000177, "nonpadding_tokens": 5900, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3290583930000821, "nonpadding_tokens": 9340, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32515345399997386, "nonpadding_tokens": 17867, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3249322960000427, "nonpadding_tokens": 3429, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32607806800001526, "nonpadding_tokens": 10550, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32971160599993254, "nonpadding_tokens": 14166, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.326286527000093, "nonpadding_tokens": 17871, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32598200299992186, "nonpadding_tokens": 9725, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003968190000023242, "prefetch": {"data_wait_seconds": 0.0024187850002590494, "consumed_batches": 245, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 250, "tokens": 22589295, "loss_tokens": 94880, "loss": 4.969270394182125, "lr": 0.00029999981990245134, "optimizer_lrs": [0.019999987993496757, 0.00029999981990245134], "gradient_norm": 0.6955717206001282, "seconds": 0.32897320500001115, "nonpadding_tokens_per_second": 288412.54715561646, "ranks": [{"compute_seconds": 0.3232768829999486, "nonpadding_tokens": 14782, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3258285470000146, "nonpadding_tokens": 6339, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32421992800004773, "nonpadding_tokens": 11385, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32203648900008375, "nonpadding_tokens": 8738, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32555688100001134, "nonpadding_tokens": 17960, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3265959049999765, "nonpadding_tokens": 13406, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32265567700005704, "nonpadding_tokens": 8501, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3247521550000556, "nonpadding_tokens": 13769, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00041651999993064237, "prefetch": {"data_wait_seconds": 0.0024285640002972286, "consumed_batches": 246, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 251, "tokens": 22681191, "loss_tokens": 91896, "loss": 4.775657400757378, "lr": 0.0002999998068920137, "optimizer_lrs": [0.01999998712613425, 0.0002999998068920137], "gradient_norm": 0.6776026487350464, "seconds": 0.5039259349999838, "nonpadding_tokens_per_second": 182360.1319507458, "ranks": [{"compute_seconds": 0.4993021809999618, "nonpadding_tokens": 6249, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4969880570000669, "nonpadding_tokens": 5028, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4990437240001029, "nonpadding_tokens": 15822, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.5004773570000225, "nonpadding_tokens": 10991, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.49773357599997325, "nonpadding_tokens": 23616, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.5014570040000308, "nonpadding_tokens": 12934, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.49967933299990364, "nonpadding_tokens": 11608, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.49756483999999546, "nonpadding_tokens": 5648, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037743899997622066, "prefetch": {"data_wait_seconds": 0.0024381370002402036, "consumed_batches": 247, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 252, "tokens": 22776280, "loss_tokens": 95089, "loss": 5.150575053371052, "lr": 0.0002999997929518994, "optimizer_lrs": [0.019999986196793294, 0.0002999997929518994], "gradient_norm": 0.7889015674591064, "seconds": 0.35507932999996683, "nonpadding_tokens_per_second": 267796.49494102877, "ranks": [{"compute_seconds": 0.35192833000007795, "nonpadding_tokens": 5258, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3482995409999603, "nonpadding_tokens": 12430, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3479560319999564, "nonpadding_tokens": 18331, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35276267500000813, "nonpadding_tokens": 19141, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34726878100002523, "nonpadding_tokens": 17873, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3500689019999754, "nonpadding_tokens": 12348, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.34747783099999197, "nonpadding_tokens": 6229, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34632526300003974, "nonpadding_tokens": 3479, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003438240000832593, "prefetch": {"data_wait_seconds": 0.002447406000214869, "consumed_batches": 248, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 253, "tokens": 22874139, "loss_tokens": 97859, "loss": 5.282211523722907, "lr": 0.0002999997780984942, "optimizer_lrs": [0.01999998520656628, 0.0002999997780984942], "gradient_norm": 0.6139222383499146, "seconds": 0.32375281199995243, "nonpadding_tokens_per_second": 302264.55608365306, "ranks": [{"compute_seconds": 0.31822444300007646, "nonpadding_tokens": 12078, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32082641199997397, "nonpadding_tokens": 9919, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31755819799991514, "nonpadding_tokens": 2194, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3207585330000029, "nonpadding_tokens": 4274, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3197912379999934, "nonpadding_tokens": 11723, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3213341669999181, "nonpadding_tokens": 14481, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3174629669999831, "nonpadding_tokens": 20144, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.317338919000008, "nonpadding_tokens": 23046, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003612409999504962, "prefetch": {"data_wait_seconds": 0.0024562680001736226, "consumed_batches": 249, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 254, "tokens": 22944696, "loss_tokens": 70557, "loss": 4.9825114800799355, "lr": 0.0002999997670698852, "optimizer_lrs": [0.019999984471325682, 0.0002999997670698852], "gradient_norm": 0.627697229385376, "seconds": 0.2909368959999483, "nonpadding_tokens_per_second": 242516.50777223022, "ranks": [{"compute_seconds": 0.2884876580000082, "nonpadding_tokens": 4964, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2840994280001041, "nonpadding_tokens": 7300, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2881002049999779, "nonpadding_tokens": 5663, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.28475168199997825, "nonpadding_tokens": 10282, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.28582559199992374, "nonpadding_tokens": 13227, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.287578860999929, "nonpadding_tokens": 3605, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.28792254800009687, "nonpadding_tokens": 8331, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.28757236499996, "nonpadding_tokens": 17185, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036518300009902305, "prefetch": {"data_wait_seconds": 0.0024655110001958747, "consumed_batches": 250, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 255, "tokens": 23058536, "loss_tokens": 113840, "loss": 4.82942342322558, "lr": 0.0002999997487119181, "optimizer_lrs": [0.019999983247461207, 0.0002999997487119181], "gradient_norm": 0.748314619064331, "seconds": 0.40497614199989584, "nonpadding_tokens_per_second": 281102.97914791555, "ranks": [{"compute_seconds": 0.40078304700000444, "nonpadding_tokens": 4811, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3964503040000409, "nonpadding_tokens": 2806, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.400397921999911, "nonpadding_tokens": 15663, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4025714549999293, "nonpadding_tokens": 4134, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4014685829999962, "nonpadding_tokens": 20114, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.40221067800007404, "nonpadding_tokens": 28016, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.40123554399997374, "nonpadding_tokens": 25062, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.39744999700008066, "nonpadding_tokens": 13234, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040502500007733033, "prefetch": {"data_wait_seconds": 0.002475407000247287, "consumed_batches": 251, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 256, "tokens": 23204127, "loss_tokens": 145591, "loss": 4.816948678146314, "lr": 0.00029999972421913585, "optimizer_lrs": [0.019999981614609058, 0.00029999972421913585], "gradient_norm": 0.7980169653892517, "seconds": 0.36407112300003064, "nonpadding_tokens_per_second": 399897.1376809463, "ranks": [{"compute_seconds": 0.3146821359999876, "nonpadding_tokens": 29687, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3417102619999923, "nonpadding_tokens": 9645, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34139324799991755, "nonpadding_tokens": 16848, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3378522130000192, "nonpadding_tokens": 15522, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3393408309999586, "nonpadding_tokens": 20331, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3436355599999388, "nonpadding_tokens": 16168, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3410386670000207, "nonpadding_tokens": 13932, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3392757920000804, "nonpadding_tokens": 23458, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.007145790999970814, "prefetch": {"data_wait_seconds": 0.0024846740002431034, "consumed_batches": 252, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 257, "tokens": 23295092, "loss_tokens": 90965, "loss": 5.029544673775628, "lr": 0.0002999997083380562, "optimizer_lrs": [0.019999980555870413, 0.0002999997083380562], "gradient_norm": 1.9321221113204956, "seconds": 0.29093827100007275, "nonpadding_tokens_per_second": 312660.82556728076, "ranks": [{"compute_seconds": 0.2872145520000231, "nonpadding_tokens": 6893, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2883605560000433, "nonpadding_tokens": 3926, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2874580769999966, "nonpadding_tokens": 9125, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.28424185899996246, "nonpadding_tokens": 7153, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.28445103999990806, "nonpadding_tokens": 7884, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2845645910000485, "nonpadding_tokens": 24951, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.28529820799997196, "nonpadding_tokens": 6402, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2875471300000072, "nonpadding_tokens": 24631, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035953700000845856, "prefetch": {"data_wait_seconds": 0.002493514000320829, "consumed_batches": 253, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 258, "tokens": 23392206, "loss_tokens": 97114, "loss": 4.641574592746669, "lr": 0.00029999969089281297, "optimizer_lrs": [0.019999979392854198, 0.00029999969089281297], "gradient_norm": 0.549118161201477, "seconds": 0.3454183810000586, "nonpadding_tokens_per_second": 281148.90620132786, "ranks": [{"compute_seconds": 0.34280935900005716, "nonpadding_tokens": 13828, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3392190289999917, "nonpadding_tokens": 18249, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.339923874999954, "nonpadding_tokens": 4616, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.338647012000024, "nonpadding_tokens": 11060, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34270969199997126, "nonpadding_tokens": 8758, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3396751899999799, "nonpadding_tokens": 11539, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.34049815700007, "nonpadding_tokens": 21542, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34036455799991927, "nonpadding_tokens": 7522, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035644499996578816, "prefetch": {"data_wait_seconds": 0.0025024720002875256, "consumed_batches": 254, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 259, "tokens": 23499419, "loss_tokens": 107213, "loss": 4.815988091462789, "lr": 0.000299999671044958, "optimizer_lrs": [0.019999978069663867, 0.000299999671044958], "gradient_norm": 0.5985225439071655, "seconds": 0.29181721499992364, "nonpadding_tokens_per_second": 367397.7904286012, "ranks": [{"compute_seconds": 0.28551341700006105, "nonpadding_tokens": 9061, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2882867079999869, "nonpadding_tokens": 10605, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2845120440000528, "nonpadding_tokens": 1530, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.28837011099994925, "nonpadding_tokens": 11470, "peak_allocated_bytes": 14992570880, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.28474346100006187, "nonpadding_tokens": 25292, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.28933437499995307, "nonpadding_tokens": 17903, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.28452060200004325, "nonpadding_tokens": 11910, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.28474314799996137, "nonpadding_tokens": 19442, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003985229999443618, "prefetch": {"data_wait_seconds": 0.002510572000232969, "consumed_batches": 255, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 260, "tokens": 23605934, "loss_tokens": 106515, "loss": 4.609645061728395, "lr": 0.0002999996507147932, "optimizer_lrs": [0.019999976714319547, 0.0002999996507147932], "gradient_norm": 0.8289932012557983, "seconds": 0.40941914699999415, "nonpadding_tokens_per_second": 260161.2571871279, "ranks": [{"compute_seconds": 0.4064597379999668, "nonpadding_tokens": 25973, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40395561900004395, "nonpadding_tokens": 17052, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.40697188599995116, "nonpadding_tokens": 12783, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.40375872800007073, "nonpadding_tokens": 15606, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4065322369999649, "nonpadding_tokens": 14095, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4041374009999572, "nonpadding_tokens": 11665, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.40517964999992273, "nonpadding_tokens": 2609, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.406814633999943, "nonpadding_tokens": 6732, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003864100000328108, "prefetch": {"data_wait_seconds": 0.002519074000247201, "consumed_batches": 256, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 261, "tokens": 23681625, "loss_tokens": 75691, "loss": 4.831556096497602, "lr": 0.0002999996358974351, "optimizer_lrs": [0.019999975726495672, 0.0002999996358974351], "gradient_norm": 0.5561065673828125, "seconds": 0.388115254000013, "nonpadding_tokens_per_second": 195021.96633579742, "ranks": [{"compute_seconds": 0.38344093099999554, "nonpadding_tokens": 12066, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3856862820000515, "nonpadding_tokens": 8245, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.38208117199997105, "nonpadding_tokens": 7581, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3853469869999344, "nonpadding_tokens": 2810, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3845955159999903, "nonpadding_tokens": 5906, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.38553554300006, "nonpadding_tokens": 13890, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3845013080000399, "nonpadding_tokens": 16274, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3812701649999326, "nonpadding_tokens": 8919, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004232129999763856, "prefetch": {"data_wait_seconds": 0.002530742000203645, "consumed_batches": 257, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 262, "tokens": 23767421, "loss_tokens": 85796, "loss": 4.252017867965873, "lr": 0.0002999996187297345, "optimizer_lrs": [0.019999974581982303, 0.0002999996187297345], "gradient_norm": 0.5631537437438965, "seconds": 0.38198610500000996, "nonpadding_tokens_per_second": 224605.02849965644, "ranks": [{"compute_seconds": 0.37620125699993423, "nonpadding_tokens": 18539, "peak_allocated_bytes": 15004887552, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3796562580000682, "nonpadding_tokens": 10393, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3783379789999799, "nonpadding_tokens": 3526, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3796836679999842, "nonpadding_tokens": 10746, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37842627700001685, "nonpadding_tokens": 6404, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3762216549999948, "nonpadding_tokens": 12268, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3764231599999448, "nonpadding_tokens": 12935, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37633210799992867, "nonpadding_tokens": 10985, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003667170000198894, "prefetch": {"data_wait_seconds": 0.002541595000138841, "consumed_batches": 258, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 263, "tokens": 23834024, "loss_tokens": 66603, "loss": 4.764478983679414, "lr": 0.0002999996051298768, "optimizer_lrs": [0.01999997367532512, 0.0002999996051298768], "gradient_norm": 0.6308606266975403, "seconds": 0.3154310740000028, "nonpadding_tokens_per_second": 211149.1399861239, "ranks": [{"compute_seconds": 0.3130231829999275, "nonpadding_tokens": 12836, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30765809299998637, "nonpadding_tokens": 12731, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30965328799993586, "nonpadding_tokens": 6584, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3131176859999414, "nonpadding_tokens": 2392, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3118997760000184, "nonpadding_tokens": 15737, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31247072799999387, "nonpadding_tokens": 11356, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31187892700006614, "nonpadding_tokens": 3442, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31196558699991783, "nonpadding_tokens": 1525, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003861719999349589, "prefetch": {"data_wait_seconds": 0.0025504880001108177, "consumed_batches": 259, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 264, "tokens": 23916096, "loss_tokens": 82072, "loss": 4.751507822399844, "lr": 0.000299999588043584, "optimizer_lrs": [0.019999972536238933, 0.000299999588043584], "gradient_norm": 0.6601517200469971, "seconds": 0.35201233899999806, "nonpadding_tokens_per_second": 233150.91804211002, "ranks": [{"compute_seconds": 0.34969601599993894, "nonpadding_tokens": 14821, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34937042600006407, "nonpadding_tokens": 4141, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3463319820000379, "nonpadding_tokens": 1044, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3493531990000065, "nonpadding_tokens": 10353, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34780809500000487, "nonpadding_tokens": 13239, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3484015220000174, "nonpadding_tokens": 12804, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3445916480000051, "nonpadding_tokens": 12378, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3480168229999663, "nonpadding_tokens": 13292, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039945200001056946, "prefetch": {"data_wait_seconds": 0.002561672000069848, "consumed_batches": 260, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 265, "tokens": 24030631, "loss_tokens": 114535, "loss": 4.699771903784869, "lr": 0.0002999995635940328, "optimizer_lrs": [0.019999970906268854, 0.0002999995635940328], "gradient_norm": 0.648922324180603, "seconds": 0.29892023499996867, "nonpadding_tokens_per_second": 383162.41789389733, "ranks": [{"compute_seconds": 0.293591398999979, "nonpadding_tokens": 9053, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2959635669999443, "nonpadding_tokens": 23379, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.29470154700004514, "nonpadding_tokens": 25266, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2966251740000416, "nonpadding_tokens": 15864, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.29224356199995327, "nonpadding_tokens": 4024, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.29171049699994, "nonpadding_tokens": 3979, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.29467823099992074, "nonpadding_tokens": 13124, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.29527873699998963, "nonpadding_tokens": 19846, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003545049999047478, "prefetch": {"data_wait_seconds": 0.002571319000026051, "consumed_batches": 261, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 266, "tokens": 24119315, "loss_tokens": 88684, "loss": 4.637736936764242, "lr": 0.00029999954417872147, "optimizer_lrs": [0.019999969611914765, 0.00029999954417872147], "gradient_norm": 0.6494580507278442, "seconds": 0.32181497900000977, "nonpadding_tokens_per_second": 275574.49400140357, "ranks": [{"compute_seconds": 0.31746548099999927, "nonpadding_tokens": 13416, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3194878140000128, "nonpadding_tokens": 11341, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3179526330000044, "nonpadding_tokens": 395, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31474659100001645, "nonpadding_tokens": 21829, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.315699972999937, "nonpadding_tokens": 10251, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3155843109999523, "nonpadding_tokens": 18043, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3161610429999655, "nonpadding_tokens": 6031, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3181956699999091, "nonpadding_tokens": 7378, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035221500002080575, "prefetch": {"data_wait_seconds": 0.0025808139999980995, "consumed_batches": 262, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 267, "tokens": 24201643, "loss_tokens": 82328, "loss": 4.805833829316879, "lr": 0.0002999995257767129, "optimizer_lrs": [0.019999968385114197, 0.0002999995257767129], "gradient_norm": 0.7276874780654907, "seconds": 0.37012202599999, "nonpadding_tokens_per_second": 222434.74912785174, "ranks": [{"compute_seconds": 0.31703751399993507, "nonpadding_tokens": 10954, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3288684190000595, "nonpadding_tokens": 11571, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32886412699997436, "nonpadding_tokens": 4266, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3287652330000128, "nonpadding_tokens": 6289, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3256973080000307, "nonpadding_tokens": 1952, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3300302689999626, "nonpadding_tokens": 24515, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32857850100003816, "nonpadding_tokens": 5551, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32809566799994627, "nonpadding_tokens": 17230, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.022074193999969793, "prefetch": {"data_wait_seconds": 0.0025894750000361455, "consumed_batches": 263, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 268, "tokens": 24299555, "loss_tokens": 97912, "loss": 4.690602913841, "lr": 0.00029999950341730156, "optimizer_lrs": [0.019999966894486774, 0.00029999950341730156], "gradient_norm": 0.5219094157218933, "seconds": 0.3150701789999175, "nonpadding_tokens_per_second": 310762.5111039964, "ranks": [{"compute_seconds": 0.31056176100003086, "nonpadding_tokens": 15424, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3124787510000715, "nonpadding_tokens": 5775, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3117570969999406, "nonpadding_tokens": 18391, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31275603699998555, "nonpadding_tokens": 9530, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3096753439999702, "nonpadding_tokens": 14067, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3124929829999701, "nonpadding_tokens": 10967, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3086793300000181, "nonpadding_tokens": 15867, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31155939699999635, "nonpadding_tokens": 7891, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004164910000099553, "prefetch": {"data_wait_seconds": 0.0026003660000242235, "consumed_batches": 264, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 269, "tokens": 24366318, "loss_tokens": 66763, "loss": 4.966994255800368, "lr": 0.00029999948787581864, "optimizer_lrs": [0.01999996585838791, 0.00029999948787581864], "gradient_norm": 2.9353630542755127, "seconds": 0.3154361950000748, "nonpadding_tokens_per_second": 211652.94616866705, "ranks": [{"compute_seconds": 0.23853184099993996, "nonpadding_tokens": 11567, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.24890642999991996, "nonpadding_tokens": 12369, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.24884398200003943, "nonpadding_tokens": 3086, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.25205255900004886, "nonpadding_tokens": 10198, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.24939613399999416, "nonpadding_tokens": 4554, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2482953069999212, "nonpadding_tokens": 2079, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.25193210800000543, "nonpadding_tokens": 9110, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.25492261399995186, "nonpadding_tokens": 13800, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.03089642299994466, "prefetch": {"data_wait_seconds": 0.0026100730000280237, "consumed_batches": 265, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 270, "tokens": 24438638, "loss_tokens": 72320, "loss": 4.816811134540929, "lr": 0.0002999994707705507, "optimizer_lrs": [0.019999964718036715, 0.0002999994707705507], "gradient_norm": 1.0166919231414795, "seconds": 0.3988359349999655, "nonpadding_tokens_per_second": 181327.69305254868, "ranks": [{"compute_seconds": 0.3964262139999164, "nonpadding_tokens": 16045, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3961006100000759, "nonpadding_tokens": 5455, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3941634239999985, "nonpadding_tokens": 7857, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3961849519999987, "nonpadding_tokens": 2240, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3929169799999954, "nonpadding_tokens": 5170, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3915743630000179, "nonpadding_tokens": 9190, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3950569290000203, "nonpadding_tokens": 6517, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.395196014000021, "nonpadding_tokens": 19846, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003822069999159794, "prefetch": {"data_wait_seconds": 0.002619793999997455, "consumed_batches": 266, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 271, "tokens": 24525321, "loss_tokens": 86683, "loss": 5.047703990401809, "lr": 0.0002999994498978787, "optimizer_lrs": [0.019999963326525247, 0.0002999994498978787], "gradient_norm": 0.6643761992454529, "seconds": 0.3610641550000082, "nonpadding_tokens_per_second": 240076.44846384163, "ranks": [{"compute_seconds": 0.356277705000025, "nonpadding_tokens": 12439, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3544083759999239, "nonpadding_tokens": 5494, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35768832099995507, "nonpadding_tokens": 14149, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35869164300004286, "nonpadding_tokens": 3273, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.357195203999936, "nonpadding_tokens": 18042, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3539033260000224, "nonpadding_tokens": 9941, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3549366800000371, "nonpadding_tokens": 3567, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35648832300000777, "nonpadding_tokens": 19778, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003630689999454262, "prefetch": {"data_wait_seconds": 0.0026302910000595148, "consumed_batches": 267, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 272, "tokens": 24600033, "loss_tokens": 74712, "loss": 4.973156504309883, "lr": 0.000299999431583832, "optimizer_lrs": [0.0199999621055888, 0.000299999431583832], "gradient_norm": 0.5796809792518616, "seconds": 0.3423642340000015, "nonpadding_tokens_per_second": 218223.7295266061, "ranks": [{"compute_seconds": 0.3384496679999529, "nonpadding_tokens": 9079, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33995328900005006, "nonpadding_tokens": 12953, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3393077279999943, "nonpadding_tokens": 7267, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3360906950000526, "nonpadding_tokens": 18058, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33625618600001417, "nonpadding_tokens": 3921, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3358079629999793, "nonpadding_tokens": 4927, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33897684599992317, "nonpadding_tokens": 7639, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33706636800002343, "nonpadding_tokens": 10868, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004113940000252114, "prefetch": {"data_wait_seconds": 0.0026400110000395216, "consumed_batches": 268, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 273, "tokens": 24676728, "loss_tokens": 76695, "loss": 4.575755019883956, "lr": 0.0002999994124717662, "optimizer_lrs": [0.01999996083145108, 0.0002999994124717662], "gradient_norm": 0.9214138984680176, "seconds": 0.3540745310000375, "nonpadding_tokens_per_second": 216606.93804602366, "ranks": [{"compute_seconds": 0.3228437989999975, "nonpadding_tokens": 5712, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3402046290000271, "nonpadding_tokens": 10464, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3365908630000831, "nonpadding_tokens": 1814, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3353295160000016, "nonpadding_tokens": 6231, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33702870100000837, "nonpadding_tokens": 6284, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34022265399994467, "nonpadding_tokens": 21383, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3392197699999997, "nonpadding_tokens": 8234, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3389722279999887, "nonpadding_tokens": 16573, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0034249699999691074, "prefetch": {"data_wait_seconds": 0.002648421000003509, "consumed_batches": 269, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 274, "tokens": 24766800, "loss_tokens": 90072, "loss": 5.085486124988898, "lr": 0.0002999993896227098, "optimizer_lrs": [0.019999959308180656, 0.0002999993896227098], "gradient_norm": 0.7195693850517273, "seconds": 0.3413952779999363, "nonpadding_tokens_per_second": 263834.9321281966, "ranks": [{"compute_seconds": 0.3344497409999576, "nonpadding_tokens": 2761, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33900165000000015, "nonpadding_tokens": 3613, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33439937800005737, "nonpadding_tokens": 12848, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33566566900003636, "nonpadding_tokens": 19568, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3326899219999859, "nonpadding_tokens": 18262, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33575168400000166, "nonpadding_tokens": 4694, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3324723030000314, "nonpadding_tokens": 14029, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33232275600005323, "nonpadding_tokens": 14297, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000444661999949858, "prefetch": {"data_wait_seconds": 0.0026614460000473628, "consumed_batches": 270, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 275, "tokens": 24845289, "loss_tokens": 78489, "loss": 4.780000621106142, "lr": 0.0002999993693565834, "optimizer_lrs": [0.019999957957105564, 0.0002999993693565834], "gradient_norm": 0.7370194792747498, "seconds": 0.32120947100008834, "nonpadding_tokens_per_second": 244354.5632562572, "ranks": [{"compute_seconds": 0.31761769900003856, "nonpadding_tokens": 11421, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.318794822999962, "nonpadding_tokens": 10951, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3145789299999251, "nonpadding_tokens": 1872, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31781216400008816, "nonpadding_tokens": 8239, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31665079100002913, "nonpadding_tokens": 18690, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3182651280000073, "nonpadding_tokens": 11590, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3177430500001037, "nonpadding_tokens": 9783, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3146107559999791, "nonpadding_tokens": 5943, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003735919999598991, "prefetch": {"data_wait_seconds": 0.002670825999985027, "consumed_batches": 271, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 276, "tokens": 24946448, "loss_tokens": 101159, "loss": 5.002513061121601, "lr": 0.0002999993427488176, "optimizer_lrs": [0.019999956183254508, 0.0002999993427488176], "gradient_norm": 0.5859196186065674, "seconds": 0.3501095929999565, "nonpadding_tokens_per_second": 288935.242057228, "ranks": [{"compute_seconds": 0.3435347390000061, "nonpadding_tokens": 18697, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34602729499999896, "nonpadding_tokens": 22529, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.344238341999926, "nonpadding_tokens": 6236, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3415846130000091, "nonpadding_tokens": 2982, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.341990686000031, "nonpadding_tokens": 22654, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34137008599998353, "nonpadding_tokens": 3552, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3442521019999276, "nonpadding_tokens": 15867, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3478428290000011, "nonpadding_tokens": 8642, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000349073000052158, "prefetch": {"data_wait_seconds": 0.0026820379999890065, "consumed_batches": 272, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 277, "tokens": 25014800, "loss_tokens": 68352, "loss": 4.691728570488062, "lr": 0.00029999932445901165, "optimizer_lrs": [0.01999995496393411, 0.00029999932445901165], "gradient_norm": 0.5942392349243164, "seconds": 0.3486160200000086, "nonpadding_tokens_per_second": 196066.72120230825, "ranks": [{"compute_seconds": 0.3452055359999804, "nonpadding_tokens": 2600, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3459808250000833, "nonpadding_tokens": 7892, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3416412029999947, "nonpadding_tokens": 6945, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34145794199991997, "nonpadding_tokens": 7968, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3446414720000348, "nonpadding_tokens": 8220, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3416648149999446, "nonpadding_tokens": 17640, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3454398480000691, "nonpadding_tokens": 6083, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34465984400003435, "nonpadding_tokens": 11004, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035425999999461055, "prefetch": {"data_wait_seconds": 0.0026911639999980252, "consumed_batches": 273, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 278, "tokens": 25111225, "loss_tokens": 96425, "loss": 5.002774176821363, "lr": 0.000299999298230555, "optimizer_lrs": [0.019999953215370337, 0.000299999298230555], "gradient_norm": 0.47125014662742615, "seconds": 0.6076467819999607, "nonpadding_tokens_per_second": 158685.93870049695, "ranks": [{"compute_seconds": 0.5742147409999916, "nonpadding_tokens": 6805, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.5705915300000015, "nonpadding_tokens": 21591, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.6043919239999695, "nonpadding_tokens": 7057, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.5750251440000511, "nonpadding_tokens": 18642, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.5723137679999581, "nonpadding_tokens": 7611, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.5763135849999799, "nonpadding_tokens": 7188, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.6043406190000269, "nonpadding_tokens": 12133, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.5744386999999733, "nonpadding_tokens": 15398, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004072519999454016, "prefetch": {"data_wait_seconds": 0.0027009770000177014, "consumed_batches": 274, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 279, "tokens": 25193201, "loss_tokens": 81976, "loss": 5.018551542524642, "lr": 0.0002999992755395048, "optimizer_lrs": [0.019999951702633655, 0.0002999992755395048], "gradient_norm": 0.6208903789520264, "seconds": 0.400458867999987, "nonpadding_tokens_per_second": 204705.16837200534, "ranks": [{"compute_seconds": 0.3942591969999967, "nonpadding_tokens": 13724, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39273486399997637, "nonpadding_tokens": 4362, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.39306249100002333, "nonpadding_tokens": 19730, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3929580989999977, "nonpadding_tokens": 5018, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39389867700003833, "nonpadding_tokens": 14767, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.39792267199993603, "nonpadding_tokens": 3321, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.39585552100004406, "nonpadding_tokens": 5666, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.39570365200006563, "nonpadding_tokens": 15388, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000461467000036464, "prefetch": {"data_wait_seconds": 0.002711857000008422, "consumed_batches": 275, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 280, "tokens": 25281965, "loss_tokens": 88764, "loss": 4.6746020346086254, "lr": 0.0002999992505624158, "optimizer_lrs": [0.01999995003749439, 0.0002999992505624158], "gradient_norm": 0.7806809544563293, "seconds": 0.4871623799999725, "nonpadding_tokens_per_second": 182206.18759602294, "ranks": [{"compute_seconds": 0.4844794699999966, "nonpadding_tokens": 26409, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4845255789999783, "nonpadding_tokens": 7142, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.48089274799997384, "nonpadding_tokens": 14152, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.48474587800001245, "nonpadding_tokens": 13904, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4806067900000244, "nonpadding_tokens": 1913, "peak_allocated_bytes": 14985477632, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4801723819999779, "nonpadding_tokens": 11325, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4837122300000374, "nonpadding_tokens": 4435, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4809414370000695, "nonpadding_tokens": 9484, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004030979999924966, "prefetch": {"data_wait_seconds": 0.002746297000044251, "consumed_batches": 276, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 281, "tokens": 25382115, "loss_tokens": 100150, "loss": 4.901865327009486, "lr": 0.00029999922187322087, "optimizer_lrs": [0.019999948124881393, 0.00029999922187322087], "gradient_norm": 0.5716951489448547, "seconds": 0.32331137400001353, "nonpadding_tokens_per_second": 309763.3057598395, "ranks": [{"compute_seconds": 0.3178565800000115, "nonpadding_tokens": 15343, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3205926470000122, "nonpadding_tokens": 9346, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31753610699990986, "nonpadding_tokens": 6453, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3209845879999875, "nonpadding_tokens": 20997, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3197634289999769, "nonpadding_tokens": 16543, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.317101426000022, "nonpadding_tokens": 9066, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31978677099994, "nonpadding_tokens": 10820, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31990637500007324, "nonpadding_tokens": 11582, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036059199999272096, "prefetch": {"data_wait_seconds": 0.0027555300000585703, "consumed_batches": 277, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 282, "tokens": 25491533, "loss_tokens": 109418, "loss": 4.553534381911568, "lr": 0.0002999991899131287, "optimizer_lrs": [0.01999994599420858, 0.0002999991899131287], "gradient_norm": 0.642160952091217, "seconds": 0.3441224549999333, "nonpadding_tokens_per_second": 317962.3951015379, "ranks": [{"compute_seconds": 0.32469625499993526, "nonpadding_tokens": 5843, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32643217299994376, "nonpadding_tokens": 16462, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32984392299999854, "nonpadding_tokens": 20572, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32678233900003306, "nonpadding_tokens": 8384, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32966432900002474, "nonpadding_tokens": 6176, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3317672110000558, "nonpadding_tokens": 17042, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32940876899999694, "nonpadding_tokens": 16078, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32897735100004866, "nonpadding_tokens": 18861, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003861000000142667, "prefetch": {"data_wait_seconds": 0.0027659340000809607, "consumed_batches": 278, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 283, "tokens": 25619220, "loss_tokens": 127687, "loss": 4.8934459851042, "lr": 0.0002999991518035524, "optimizer_lrs": [0.01999994345357016, 0.0002999991518035524], "gradient_norm": 0.5788370966911316, "seconds": 0.40159637099998235, "nonpadding_tokens_per_second": 317948.59022768814, "ranks": [{"compute_seconds": 0.396771565999984, "nonpadding_tokens": 19731, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3939772600000424, "nonpadding_tokens": 19578, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.39785144400002537, "nonpadding_tokens": 6655, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.39924597900005665, "nonpadding_tokens": 8461, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3948330700000042, "nonpadding_tokens": 17049, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.394891862999998, "nonpadding_tokens": 13269, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3950423150000688, "nonpadding_tokens": 18928, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.39502230900006907, "nonpadding_tokens": 24016, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036617400007799006, "prefetch": {"data_wait_seconds": 0.0027759670000477854, "consumed_batches": 279, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 284, "tokens": 25695115, "loss_tokens": 75895, "loss": 4.593428420844588, "lr": 0.0002999991287368183, "optimizer_lrs": [0.01999994191578789, 0.0002999991287368183], "gradient_norm": 0.6740460395812988, "seconds": 0.6012363439999717, "nonpadding_tokens_per_second": 126231.55728590414, "ranks": [{"compute_seconds": 0.5839092170000413, "nonpadding_tokens": 3510, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.5882646860000023, "nonpadding_tokens": 7754, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.5917517340000131, "nonpadding_tokens": 4940, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.594620756999916, "nonpadding_tokens": 6225, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.5933992459999899, "nonpadding_tokens": 8098, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.5911240469999939, "nonpadding_tokens": 19929, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.5948854539999502, "nonpadding_tokens": 13613, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.5933170990000463, "nonpadding_tokens": 11826, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0024679580000110946, "prefetch": {"data_wait_seconds": 0.002784540000106972, "consumed_batches": 280, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 285, "tokens": 25852341, "loss_tokens": 157226, "loss": 4.601106846195922, "lr": 0.00029999907996661635, "optimizer_lrs": [0.01999993866444109, 0.00029999907996661635], "gradient_norm": 0.6963496804237366, "seconds": 0.3999740419999398, "nonpadding_tokens_per_second": 393090.5096086802, "ranks": [{"compute_seconds": 0.39498265500003527, "nonpadding_tokens": 26235, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39281179600004634, "nonpadding_tokens": 18598, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3931708569999728, "nonpadding_tokens": 26209, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.39262662299995554, "nonpadding_tokens": 28635, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39293616000009024, "nonpadding_tokens": 14619, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3975870650000388, "nonpadding_tokens": 19805, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3953535499999816, "nonpadding_tokens": 18819, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3952482840001039, "nonpadding_tokens": 4306, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003852639999877283, "prefetch": {"data_wait_seconds": 0.0027939960001504005, "consumed_batches": 281, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 286, "tokens": 25958342, "loss_tokens": 106001, "loss": 4.973232563843737, "lr": 0.0002999990463364664, "optimizer_lrs": [0.019999936422431096, 0.0002999990463364664], "gradient_norm": 0.7160050868988037, "seconds": 0.33591344699993897, "nonpadding_tokens_per_second": 315560.45447629626, "ranks": [{"compute_seconds": 0.3315316810000013, "nonpadding_tokens": 16914, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.333410904999937, "nonpadding_tokens": 18817, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32950374400002147, "nonpadding_tokens": 24745, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32851920600000994, "nonpadding_tokens": 13965, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.329459810000003, "nonpadding_tokens": 2176, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3335369990001027, "nonpadding_tokens": 7571, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3319520229999853, "nonpadding_tokens": 10701, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33128321200001665, "nonpadding_tokens": 11112, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036364499999308464, "prefetch": {"data_wait_seconds": 0.0028031000000510176, "consumed_batches": 282, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 287, "tokens": 26062448, "loss_tokens": 104106, "loss": 4.65206196088602, "lr": 0.00029999901271996116, "optimizer_lrs": [0.019999934181330746, 0.00029999901271996116], "gradient_norm": 0.5890917778015137, "seconds": 0.3369278820000545, "nonpadding_tokens_per_second": 308986.0043105104, "ranks": [{"compute_seconds": 0.33269581000001835, "nonpadding_tokens": 7083, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32975245299996914, "nonpadding_tokens": 7836, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33081190400002924, "nonpadding_tokens": 25052, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33450391999997464, "nonpadding_tokens": 26240, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3326615910000328, "nonpadding_tokens": 11372, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33376443600002403, "nonpadding_tokens": 6107, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33122002099992187, "nonpadding_tokens": 15105, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33104602600008093, "nonpadding_tokens": 5311, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003884879999986879, "prefetch": {"data_wait_seconds": 0.002813249000041651, "consumed_batches": 283, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 288, "tokens": 26167695, "loss_tokens": 105247, "loss": 4.984205012969491, "lr": 0.0002999989781431438, "optimizer_lrs": [0.019999931876209586, 0.0002999989781431438], "gradient_norm": 0.5990774631500244, "seconds": 0.33023678100005327, "nonpadding_tokens_per_second": 318701.6288170003, "ranks": [{"compute_seconds": 0.3062946170000487, "nonpadding_tokens": 1548, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3205355169999393, "nonpadding_tokens": 16800, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3168748810001034, "nonpadding_tokens": 6000, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32106327999997575, "nonpadding_tokens": 16936, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31715233199997783, "nonpadding_tokens": 24070, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32150519300000724, "nonpadding_tokens": 12336, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3200391990000071, "nonpadding_tokens": 10766, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31983553899999606, "nonpadding_tokens": 16791, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00041045699992992013, "prefetch": {"data_wait_seconds": 0.002824214000042957, "consumed_batches": 284, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 289, "tokens": 26275959, "loss_tokens": 108264, "loss": 4.661642201008646, "lr": 0.00029999894195421623, "optimizer_lrs": [0.019999929463614417, 0.00029999894195421623], "gradient_norm": 0.5979421138763428, "seconds": 0.3805867370000442, "nonpadding_tokens_per_second": 284466.03487390425, "ranks": [{"compute_seconds": 0.37699972799998704, "nonpadding_tokens": 9223, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3780012890000535, "nonpadding_tokens": 12204, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.37684478400001353, "nonpadding_tokens": 19711, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37747463100004097, "nonpadding_tokens": 25131, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3743598449999581, "nonpadding_tokens": 12374, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37665286499998274, "nonpadding_tokens": 6509, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.37691213799996603, "nonpadding_tokens": 6345, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3764575349999859, "nonpadding_tokens": 16767, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00041600899999139074, "prefetch": {"data_wait_seconds": 0.0028351859999702356, "consumed_batches": 285, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 290, "tokens": 26340453, "loss_tokens": 64494, "loss": 4.276264555617577, "lr": 0.00029999892009679806, "optimizer_lrs": [0.019999928006453206, 0.00029999892009679806], "gradient_norm": 1.8487931489944458, "seconds": 0.3169181769999341, "nonpadding_tokens_per_second": 203503.63179078055, "ranks": [{"compute_seconds": 0.3124628269999903, "nonpadding_tokens": 2393, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3143233609999925, "nonpadding_tokens": 11890, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3124458610000147, "nonpadding_tokens": 11704, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30933101599998736, "nonpadding_tokens": 12252, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31287638900005277, "nonpadding_tokens": 3497, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30962799999997515, "nonpadding_tokens": 11843, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31289217799997004, "nonpadding_tokens": 8266, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3105514830000402, "nonpadding_tokens": 2649, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040577599997959624, "prefetch": {"data_wait_seconds": 0.0028449440000031245, "consumed_batches": 286, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 291, "tokens": 26410546, "loss_tokens": 70093, "loss": 4.524873204171601, "lr": 0.0002999988960884365, "optimizer_lrs": [0.01999992640589577, 0.0002999988960884365], "gradient_norm": 0.6520408987998962, "seconds": 0.3065862680000464, "nonpadding_tokens_per_second": 228624.06870743926, "ranks": [{"compute_seconds": 0.3031519119999757, "nonpadding_tokens": 5301, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3026711399999158, "nonpadding_tokens": 3680, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30265389300006973, "nonpadding_tokens": 2622, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3020552680000037, "nonpadding_tokens": 13288, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3043414679999614, "nonpadding_tokens": 9881, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30235804200003713, "nonpadding_tokens": 9944, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.30352201599998807, "nonpadding_tokens": 13125, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3033371020000004, "nonpadding_tokens": 12252, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003831779999927676, "prefetch": {"data_wait_seconds": 0.0028550319999567364, "consumed_batches": 287, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 292, "tokens": 26521488, "loss_tokens": 110942, "loss": 4.429396666726758, "lr": 0.0002999988575489027, "optimizer_lrs": [0.019999923836593516, 0.0002999988575489027], "gradient_norm": 0.7185731530189514, "seconds": 0.35242469500008156, "nonpadding_tokens_per_second": 314796.3283332751, "ranks": [{"compute_seconds": 0.3500584380000191, "nonpadding_tokens": 19663, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34974325099994985, "nonpadding_tokens": 20348, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3463661979999415, "nonpadding_tokens": 17273, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34999697099999594, "nonpadding_tokens": 12483, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34852679899995564, "nonpadding_tokens": 5882, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34913778200007073, "nonpadding_tokens": 14538, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.34812240899998415, "nonpadding_tokens": 3522, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34605817100009517, "nonpadding_tokens": 17233, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035292500001560256, "prefetch": {"data_wait_seconds": 0.002864176999992196, "consumed_batches": 288, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 293, "tokens": 26633601, "loss_tokens": 112113, "loss": 4.60877975792281, "lr": 0.00029999881793082945, "optimizer_lrs": [0.019999921195388633, 0.00029999881793082945], "gradient_norm": 0.7568525075912476, "seconds": 0.25761015300008694, "nonpadding_tokens_per_second": 435204.1202349744, "ranks": [{"compute_seconds": 0.25267409200000657, "nonpadding_tokens": 10183, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2510449979999976, "nonpadding_tokens": 24000, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2516914349999979, "nonpadding_tokens": 7531, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2511945910000577, "nonpadding_tokens": 11706, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2539580060000617, "nonpadding_tokens": 12918, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2553200239999569, "nonpadding_tokens": 18080, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.25428781499999786, "nonpadding_tokens": 9707, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2519670420000466, "nonpadding_tokens": 17988, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039420899997821834, "prefetch": {"data_wait_seconds": 0.0028748899999300193, "consumed_batches": 289, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 294, "tokens": 26700895, "loss_tokens": 67294, "loss": 4.670145183820251, "lr": 0.00029999879382641963, "optimizer_lrs": [0.019999919588427978, 0.00029999879382641963], "gradient_norm": 0.588208794593811, "seconds": 0.37055804600004194, "nonpadding_tokens_per_second": 181601.77798431175, "ranks": [{"compute_seconds": 0.3659075010000379, "nonpadding_tokens": 7381, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3635934769999949, "nonpadding_tokens": 3816, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3644152710000981, "nonpadding_tokens": 12958, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3682724779999944, "nonpadding_tokens": 6352, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3649781519999351, "nonpadding_tokens": 5159, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3644203739999057, "nonpadding_tokens": 8733, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3647001420000606, "nonpadding_tokens": 11023, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.367877490000069, "nonpadding_tokens": 11872, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034208700003546255, "prefetch": {"data_wait_seconds": 0.002884049999920535, "consumed_batches": 290, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 295, "tokens": 26790379, "loss_tokens": 89484, "loss": 4.357896383711054, "lr": 0.00029999876139679863, "optimizer_lrs": [0.019999917426453243, 0.00029999876139679863], "gradient_norm": 0.5708726048469543, "seconds": 0.5175649650000196, "nonpadding_tokens_per_second": 172894.23753788398, "ranks": [{"compute_seconds": 0.49727562800001124, "nonpadding_tokens": 14298, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.5052727880000702, "nonpadding_tokens": 17448, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.5023511079999707, "nonpadding_tokens": 21686, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4999064939999016, "nonpadding_tokens": 10989, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.5032867369999394, "nonpadding_tokens": 8963, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.500576983999963, "nonpadding_tokens": 8677, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.5014881009999499, "nonpadding_tokens": 4205, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.5034845200000291, "nonpadding_tokens": 3218, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.008709239000040725, "prefetch": {"data_wait_seconds": 0.0028936300000168558, "consumed_batches": 291, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 296, "tokens": 26858342, "loss_tokens": 67963, "loss": 4.422100996130247, "lr": 0.00029999873647909656, "optimizer_lrs": [0.019999915765273105, 0.00029999873647909656], "gradient_norm": 0.6325992941856384, "seconds": 0.3709177069999896, "nonpadding_tokens_per_second": 183229.32207709865, "ranks": [{"compute_seconds": 0.36637671999994836, "nonpadding_tokens": 9517, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3679299399999536, "nonpadding_tokens": 7508, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3664075269999785, "nonpadding_tokens": 9423, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3684144080000351, "nonpadding_tokens": 5774, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3662676689999671, "nonpadding_tokens": 1982, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36836508400006096, "nonpadding_tokens": 2052, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.36610733400004847, "nonpadding_tokens": 14092, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36677198499990027, "nonpadding_tokens": 17615, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00042382000003726716, "prefetch": {"data_wait_seconds": 0.0029039390000207277, "consumed_batches": 292, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 297, "tokens": 26922435, "loss_tokens": 64093, "loss": 4.415709398842307, "lr": 0.0002999987127529194, "optimizer_lrs": [0.01999991418352796, 0.0002999987127529194], "gradient_norm": 0.5685275793075562, "seconds": 0.40313993699999173, "nonpadding_tokens_per_second": 158984.49674064745, "ranks": [{"compute_seconds": 0.37358343100004276, "nonpadding_tokens": 20786, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3775575449999451, "nonpadding_tokens": 9125, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.37617980300001364, "nonpadding_tokens": 2844, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.376732605999905, "nonpadding_tokens": 6751, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3741848160000245, "nonpadding_tokens": 6535, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37325717900000654, "nonpadding_tokens": 4330, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.37571803999992426, "nonpadding_tokens": 1496, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37428395699998873, "nonpadding_tokens": 12226, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.02361578299996836, "prefetch": {"data_wait_seconds": 0.0029139619999796196, "consumed_batches": 293, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 298, "tokens": 27025142, "loss_tokens": 102707, "loss": 4.7595252757845135, "lr": 0.0002999986742722832, "optimizer_lrs": [0.019999911618152212, 0.0002999986742722832], "gradient_norm": 0.7204530835151672, "seconds": 0.4496885089999978, "nonpadding_tokens_per_second": 228395.87390924527, "ranks": [{"compute_seconds": 0.44611097300003166, "nonpadding_tokens": 12339, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4424579830000539, "nonpadding_tokens": 12107, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4442900159999681, "nonpadding_tokens": 19740, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4424066840000478, "nonpadding_tokens": 13695, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.446138074999908, "nonpadding_tokens": 9579, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.44713179999996555, "nonpadding_tokens": 22074, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.44731823899996925, "nonpadding_tokens": 8336, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.44712246999995386, "nonpadding_tokens": 4837, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039023199997245683, "prefetch": {"data_wait_seconds": 0.002938784999969357, "consumed_batches": 294, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 299, "tokens": 27138608, "loss_tokens": 113466, "loss": 4.575111432058943, "lr": 0.00029999863110175267, "optimizer_lrs": [0.019999908740116847, 0.00029999863110175267], "gradient_norm": 0.5223949551582336, "seconds": 0.4280751799999507, "nonpadding_tokens_per_second": 265060.9175706311, "ranks": [{"compute_seconds": 0.42507322899996325, "nonpadding_tokens": 15199, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.42568299199990633, "nonpadding_tokens": 29376, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4227189149999049, "nonpadding_tokens": 16980, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4217384940000102, "nonpadding_tokens": 12281, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.42475056000000677, "nonpadding_tokens": 4345, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4216031280000152, "nonpadding_tokens": 9972, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.42302843900006337, "nonpadding_tokens": 12014, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.42252903999997216, "nonpadding_tokens": 13299, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004009419999420061, "prefetch": {"data_wait_seconds": 0.002948199000002205, "consumed_batches": 295, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 300, "tokens": 27236850, "loss_tokens": 98242, "loss": 4.208726601148185, "lr": 0.00029999859316481864, "optimizer_lrs": [0.01999990621098791, 0.00029999859316481864], "gradient_norm": 0.5928810238838196, "seconds": 0.49789349000002403, "nonpadding_tokens_per_second": 197315.29327687185, "ranks": [{"compute_seconds": 0.4945607449999443, "nonpadding_tokens": 5250, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4946192429999883, "nonpadding_tokens": 24467, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.49169155999993563, "nonpadding_tokens": 7721, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4954895009999518, "nonpadding_tokens": 3904, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4912475790000599, "nonpadding_tokens": 19202, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.49125282999989395, "nonpadding_tokens": 13753, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4919504820001066, "nonpadding_tokens": 14929, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4942512000000079, "nonpadding_tokens": 9016, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035601499996573693, "prefetch": {"data_wait_seconds": 0.0029581750000033935, "consumed_batches": 296, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 301, "tokens": 27371045, "loss_tokens": 134195, "loss": 4.769027814002012, "lr": 0.0002999985405064659, "optimizer_lrs": [0.019999902700431064, 0.0002999985405064659], "gradient_norm": 0.6923686265945435, "seconds": 0.4721430600000076, "nonpadding_tokens_per_second": 284225.29391832603, "ranks": [{"compute_seconds": 0.4684687550000035, "nonpadding_tokens": 19928, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.46973203300001387, "nonpadding_tokens": 19690, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4649868599999536, "nonpadding_tokens": 24851, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.46868491899999754, "nonpadding_tokens": 9485, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4679061649999312, "nonpadding_tokens": 21387, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.46488765000003696, "nonpadding_tokens": 21533, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.46638135199998487, "nonpadding_tokens": 7003, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4663697130000628, "nonpadding_tokens": 10318, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004054539999742701, "prefetch": {"data_wait_seconds": 0.002968129000009867, "consumed_batches": 297, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 302, "tokens": 27495067, "loss_tokens": 124022, "loss": 4.3886729773749815, "lr": 0.0002999984909797681, "optimizer_lrs": [0.01999989939865121, 0.0002999984909797681], "gradient_norm": 0.5101412534713745, "seconds": 0.36644547599996713, "nonpadding_tokens_per_second": 338445.9847991441, "ranks": [{"compute_seconds": 0.36196531000007326, "nonpadding_tokens": 18945, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35908246599990434, "nonpadding_tokens": 17645, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3624694800000725, "nonpadding_tokens": 15101, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3631581099999721, "nonpadding_tokens": 3456, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3618669800000589, "nonpadding_tokens": 5526, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3639696239999921, "nonpadding_tokens": 25572, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3600570340000786, "nonpadding_tokens": 16342, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36210073600000214, "nonpadding_tokens": 21435, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004277420000562415, "prefetch": {"data_wait_seconds": 0.002977472999987185, "consumed_batches": 298, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 303, "tokens": 27623035, "loss_tokens": 127968, "loss": 5.104548598087022, "lr": 0.0002999984390110697, "optimizer_lrs": [0.019999895934071314, 0.0002999984390110697], "gradient_norm": 0.8755532503128052, "seconds": 0.34379635499999495, "nonpadding_tokens_per_second": 372220.35120180924, "ranks": [{"compute_seconds": 0.2619204410000293, "nonpadding_tokens": 10505, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2858305839999957, "nonpadding_tokens": 26362, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.282460158000049, "nonpadding_tokens": 16472, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.28645260600001166, "nonpadding_tokens": 13523, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2822748580000507, "nonpadding_tokens": 14216, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2821317430000363, "nonpadding_tokens": 14051, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.28555951800001367, "nonpadding_tokens": 4873, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.281358694000005, "nonpadding_tokens": 27966, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0410486990000436, "prefetch": {"data_wait_seconds": 0.0029868739999301397, "consumed_batches": 299, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 304, "tokens": 27785455, "loss_tokens": 162420, "loss": 4.610699113409678, "lr": 0.0002999983717842278, "optimizer_lrs": [0.019999891452281857, 0.0002999983717842278], "gradient_norm": 0.6005547046661377, "seconds": 0.38065990799998417, "nonpadding_tokens_per_second": 426680.08000466065, "ranks": [{"compute_seconds": 0.37637452900003154, "nonpadding_tokens": 28303, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.37305019299992637, "nonpadding_tokens": 11251, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3738944819999688, "nonpadding_tokens": 22309, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37668040699998073, "nonpadding_tokens": 22398, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37671545900002457, "nonpadding_tokens": 12901, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3783103119999396, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3745109299999285, "nonpadding_tokens": 17639, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3765898740000466, "nonpadding_tokens": 14851, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036530199997741875, "prefetch": {"data_wait_seconds": 0.002995630999862442, "consumed_batches": 300, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 305, "tokens": 27898102, "loss_tokens": 112647, "loss": 4.790117579695864, "lr": 0.00029999832432646537, "optimizer_lrs": [0.019999888288431026, 0.00029999832432646537], "gradient_norm": 0.62909334897995, "seconds": 0.33544611800004986, "nonpadding_tokens_per_second": 335812.501487894, "ranks": [{"compute_seconds": 0.3321371399999862, "nonpadding_tokens": 16051, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3315503979999903, "nonpadding_tokens": 8660, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3284877559999586, "nonpadding_tokens": 19413, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3282264639999539, "nonpadding_tokens": 25066, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32887643900005514, "nonpadding_tokens": 16661, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33302658599995993, "nonpadding_tokens": 14193, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32917430099996636, "nonpadding_tokens": 2668, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32913181399999303, "nonpadding_tokens": 9935, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003699500000493572, "prefetch": {"data_wait_seconds": 0.003004803999942851, "consumed_batches": 301, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 306, "tokens": 28022819, "loss_tokens": 124717, "loss": 4.8274493453178, "lr": 0.00029999827098845026, "optimizer_lrs": [0.019999884732563353, 0.00029999827098845026], "gradient_norm": 0.5638412833213806, "seconds": 0.43639462600003753, "nonpadding_tokens_per_second": 285789.495492067, "ranks": [{"compute_seconds": 0.43359691299997394, "nonpadding_tokens": 32768, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.43409290199997486, "nonpadding_tokens": 18829, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4332547340000019, "nonpadding_tokens": 25145, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.43323870099993655, "nonpadding_tokens": 12488, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4307078359998968, "nonpadding_tokens": 7482, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.43056529599994064, "nonpadding_tokens": 2986, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.43138251300001684, "nonpadding_tokens": 20425, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.43280840199997783, "nonpadding_tokens": 4594, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004037349999634898, "prefetch": {"data_wait_seconds": 0.0030146999999942636, "consumed_batches": 302, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 307, "tokens": 28122414, "loss_tokens": 99595, "loss": 4.646048998443697, "lr": 0.00029999822779431254, "optimizer_lrs": [0.01999988185295417, 0.00029999822779431254], "gradient_norm": 0.49205443263053894, "seconds": 0.3295006830001057, "nonpadding_tokens_per_second": 302260.37498067354, "ranks": [{"compute_seconds": 0.324919551999983, "nonpadding_tokens": 7083, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32447877500010236, "nonpadding_tokens": 12057, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32473773499998515, "nonpadding_tokens": 10109, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3243840069999351, "nonpadding_tokens": 3892, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32539532400005555, "nonpadding_tokens": 24796, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3243931810000049, "nonpadding_tokens": 7729, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32522639100000106, "nonpadding_tokens": 16299, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32722808899995925, "nonpadding_tokens": 17630, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037028500003088993, "prefetch": {"data_wait_seconds": 0.00302366300002177, "consumed_batches": 303, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 308, "tokens": 28220708, "loss_tokens": 98294, "loss": 4.6094344517468, "lr": 0.000299998184641914, "optimizer_lrs": [0.0199998789761276, 0.000299998184641914], "gradient_norm": 0.5577824711799622, "seconds": 0.3267677869999943, "nonpadding_tokens_per_second": 300806.88461498107, "ranks": [{"compute_seconds": 0.3222297199999957, "nonpadding_tokens": 8445, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32232748500007347, "nonpadding_tokens": 10823, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3226886890000742, "nonpadding_tokens": 24627, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32163473399998566, "nonpadding_tokens": 26051, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3204592820000016, "nonpadding_tokens": 2372, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3243028020000338, "nonpadding_tokens": 9946, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32042376099991543, "nonpadding_tokens": 3834, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3195716509999329, "nonpadding_tokens": 12196, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004264229999080271, "prefetch": {"data_wait_seconds": 0.0030343370000309733, "consumed_batches": 304, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 309, "tokens": 28304110, "loss_tokens": 83402, "loss": 4.348719904798446, "lr": 0.000299998147620243, "optimizer_lrs": [0.019999876508016202, 0.000299998147620243], "gradient_norm": 0.555621862411499, "seconds": 0.2872082659999933, "nonpadding_tokens_per_second": 290388.5781616117, "ranks": [{"compute_seconds": 0.2828423920000205, "nonpadding_tokens": 8662, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.282395311000073, "nonpadding_tokens": 10008, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.27973253100003603, "nonpadding_tokens": 15905, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2831466459999774, "nonpadding_tokens": 14348, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.28030917499995667, "nonpadding_tokens": 9812, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2846411390000867, "nonpadding_tokens": 2186, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.28256360600005337, "nonpadding_tokens": 11521, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2820654999999306, "nonpadding_tokens": 10960, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038067299999511306, "prefetch": {"data_wait_seconds": 0.003044433000013669, "consumed_batches": 305, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 310, "tokens": 28421084, "loss_tokens": 116974, "loss": 4.788255296048694, "lr": 0.0002999980950665399, "optimizer_lrs": [0.019999873004435995, 0.0002999980950665399], "gradient_norm": 0.5938318371772766, "seconds": 0.3679387959999758, "nonpadding_tokens_per_second": 317917.004870581, "ranks": [{"compute_seconds": 0.3629328900000246, "nonpadding_tokens": 13815, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3653349230000913, "nonpadding_tokens": 19970, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3654828159999397, "nonpadding_tokens": 28032, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36210395000000517, "nonpadding_tokens": 7300, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36293041999999787, "nonpadding_tokens": 9160, "peak_allocated_bytes": 14985618944, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3616329339999993, "nonpadding_tokens": 9801, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.36330994499996905, "nonpadding_tokens": 9147, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3651402450000205, "nonpadding_tokens": 19749, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039730399998916255, "prefetch": {"data_wait_seconds": 0.00305637700000716, "consumed_batches": 306, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 311, "tokens": 28492544, "loss_tokens": 71460, "loss": 4.3521738560033585, "lr": 0.00029999806259951015, "optimizer_lrs": [0.019999870839967345, 0.00029999806259951015], "gradient_norm": 0.615782618522644, "seconds": 0.39341178500001206, "nonpadding_tokens_per_second": 181641.7370415017, "ranks": [{"compute_seconds": 0.3910193829999571, "nonpadding_tokens": 18338, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3901376149999578, "nonpadding_tokens": 5225, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.38985461300001134, "nonpadding_tokens": 10689, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3864586900000404, "nonpadding_tokens": 2362, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3873977029999196, "nonpadding_tokens": 14893, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.38714944399998785, "nonpadding_tokens": 3986, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3899845319999713, "nonpadding_tokens": 3964, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.39008604399998603, "nonpadding_tokens": 12003, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036316299997452006, "prefetch": {"data_wait_seconds": 0.003066079000063837, "consumed_batches": 307, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 312, "tokens": 28576781, "loss_tokens": 84237, "loss": 4.721478165770386, "lr": 0.0002999980239750899, "optimizer_lrs": [0.019999868265005997, 0.0002999980239750899], "gradient_norm": 0.490523099899292, "seconds": 0.2717826100000593, "nonpadding_tokens_per_second": 309942.5677013758, "ranks": [{"compute_seconds": 0.26832919700007096, "nonpadding_tokens": 7708, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.26938010300000315, "nonpadding_tokens": 6626, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2681998369999974, "nonpadding_tokens": 4802, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2694288059999508, "nonpadding_tokens": 4188, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.268052950000083, "nonpadding_tokens": 22064, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2650832709999804, "nonpadding_tokens": 4754, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.26821044200005417, "nonpadding_tokens": 20612, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.26610775799997555, "nonpadding_tokens": 13483, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037710900005549775, "prefetch": {"data_wait_seconds": 0.003076270000065051, "consumed_batches": 308, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 313, "tokens": 28653536, "loss_tokens": 76755, "loss": 4.3620492964627715, "lr": 0.0002999979884493904, "optimizer_lrs": [0.01999986589662603, 0.0002999979884493904], "gradient_norm": 0.5627779364585876, "seconds": 0.2938640620000115, "nonpadding_tokens_per_second": 261192.19709144632, "ranks": [{"compute_seconds": 0.28997174299991, "nonpadding_tokens": 2868, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2874261300000853, "nonpadding_tokens": 14007, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2906058610000173, "nonpadding_tokens": 3119, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2905896940000048, "nonpadding_tokens": 10869, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2882996649999541, "nonpadding_tokens": 7051, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.29154260200004956, "nonpadding_tokens": 6997, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.29032771200002117, "nonpadding_tokens": 28866, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2896126069999809, "nonpadding_tokens": 2978, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036170600003515574, "prefetch": {"data_wait_seconds": 0.0030866720001085923, "consumed_batches": 309, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 314, "tokens": 28737233, "loss_tokens": 83697, "loss": 4.769059449562111, "lr": 0.0002999979493498788, "optimizer_lrs": [0.019999863289991923, 0.0002999979493498788], "gradient_norm": 0.4872666895389557, "seconds": 0.3383106219999945, "nonpadding_tokens_per_second": 247396.9026015428, "ranks": [{"compute_seconds": 0.3359514299999091, "nonpadding_tokens": 4923, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3357818289999841, "nonpadding_tokens": 5509, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3341634209999711, "nonpadding_tokens": 19386, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33557932000007895, "nonpadding_tokens": 16514, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33546814900000754, "nonpadding_tokens": 18250, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3316207039999881, "nonpadding_tokens": 8488, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33242827599997327, "nonpadding_tokens": 8088, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33394821900003535, "nonpadding_tokens": 2539, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035376400001041475, "prefetch": {"data_wait_seconds": 0.0030964920000542406, "consumed_batches": 310, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 315, "tokens": 28841192, "loss_tokens": 103959, "loss": 4.172617931107456, "lr": 0.00029999790026082496, "optimizer_lrs": [0.01999986001738833, 0.00029999790026082496], "gradient_norm": 0.65008944272995, "seconds": 0.3617991720000191, "nonpadding_tokens_per_second": 287338.9660493599, "ranks": [{"compute_seconds": 0.35947368500001176, "nonpadding_tokens": 16885, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35847136799998225, "nonpadding_tokens": 13452, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35822338100001616, "nonpadding_tokens": 10041, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3552173450000282, "nonpadding_tokens": 15742, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35805346000006466, "nonpadding_tokens": 6728, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3544748180000852, "nonpadding_tokens": 20147, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3575243479999699, "nonpadding_tokens": 7484, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35614284099995075, "nonpadding_tokens": 13480, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003638470000169036, "prefetch": {"data_wait_seconds": 0.0031073610000476037, "consumed_batches": 311, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 316, "tokens": 28904748, "loss_tokens": 63556, "loss": 4.54521406318837, "lr": 0.00029999786996392943, "optimizer_lrs": [0.019999857997595297, 0.00029999786996392943], "gradient_norm": 0.6225787997245789, "seconds": 0.3079937489999338, "nonpadding_tokens_per_second": 206354.8374159167, "ranks": [{"compute_seconds": 0.3013871670000299, "nonpadding_tokens": 3610, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3045431540000436, "nonpadding_tokens": 15592, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30313960899991343, "nonpadding_tokens": 10311, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30298076500002935, "nonpadding_tokens": 3303, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3022577110000384, "nonpadding_tokens": 4488, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30556951899995966, "nonpadding_tokens": 13793, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3041293499999256, "nonpadding_tokens": 8994, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3014313270000457, "nonpadding_tokens": 3465, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003709159999516487, "prefetch": {"data_wait_seconds": 0.0031178770000224176, "consumed_batches": 312, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 317, "tokens": 28989345, "loss_tokens": 84597, "loss": 4.739797215031266, "lr": 0.00029999782930020265, "optimizer_lrs": [0.01999985528668018, 0.00029999782930020265], "gradient_norm": 0.45564088225364685, "seconds": 0.25198089200000595, "nonpadding_tokens_per_second": 335727.83764888806, "ranks": [{"compute_seconds": 0.24840110799993909, "nonpadding_tokens": 8973, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2492497910000111, "nonpadding_tokens": 4782, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.24813219899999694, "nonpadding_tokens": 6485, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.24950896300003933, "nonpadding_tokens": 12776, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.24747289599997657, "nonpadding_tokens": 9636, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.24895907399991302, "nonpadding_tokens": 2727, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2452668219999623, "nonpadding_tokens": 21074, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.24745303600002444, "nonpadding_tokens": 18144, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003927459999886196, "prefetch": {"data_wait_seconds": 0.0031263559999388235, "consumed_batches": 313, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 318, "tokens": 29079235, "loss_tokens": 89890, "loss": 4.271085493380799, "lr": 0.00029999778567093877, "optimizer_lrs": [0.019999852378062587, 0.00029999778567093877], "gradient_norm": 0.5125133991241455, "seconds": 0.35312910599998304, "nonpadding_tokens_per_second": 254552.79237164982, "ranks": [{"compute_seconds": 0.30501378000008117, "nonpadding_tokens": 13455, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3100582609999947, "nonpadding_tokens": 10108, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31380524899998363, "nonpadding_tokens": 3537, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31423018799989677, "nonpadding_tokens": 9692, "peak_allocated_bytes": 14996700672, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3113968579999664, "nonpadding_tokens": 2435, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31582415000002584, "nonpadding_tokens": 15955, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.311475664999989, "nonpadding_tokens": 20763, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3117406229999915, "nonpadding_tokens": 13945, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004370630000494202, "prefetch": {"data_wait_seconds": 0.0031360039998844513, "consumed_batches": 314, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 319, "tokens": 29152143, "loss_tokens": 72908, "loss": 4.304260163493718, "lr": 0.0002999977499652743, "optimizer_lrs": [0.019999849997684956, 0.0002999977499652743], "gradient_norm": 0.678565263748169, "seconds": 0.336092911000037, "nonpadding_tokens_per_second": 216928.11009629408, "ranks": [{"compute_seconds": 0.33314153799994983, "nonpadding_tokens": 9818, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3336892389999093, "nonpadding_tokens": 7789, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33032647100003487, "nonpadding_tokens": 12267, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32957322799995836, "nonpadding_tokens": 17023, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33256771100002425, "nonpadding_tokens": 4758, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33364951000010024, "nonpadding_tokens": 10218, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33096115000000736, "nonpadding_tokens": 6513, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.332540052000013, "nonpadding_tokens": 4522, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000381000000061249, "prefetch": {"data_wait_seconds": 0.0031474709999201878, "consumed_batches": 315, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 320, "tokens": 29247031, "loss_tokens": 94888, "loss": 4.685040521456876, "lr": 0.00029999770306752356, "optimizer_lrs": [0.01999984687116824, 0.00029999770306752356], "gradient_norm": 0.5406339764595032, "seconds": 0.44769342699999015, "nonpadding_tokens_per_second": 211948.61098553071, "ranks": [{"compute_seconds": 0.4413046999999324, "nonpadding_tokens": 26511, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4406184399999802, "nonpadding_tokens": 10518, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.441222312999912, "nonpadding_tokens": 6288, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4452168579999807, "nonpadding_tokens": 10086, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4415265369999588, "nonpadding_tokens": 11777, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.44123034200003985, "nonpadding_tokens": 2586, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4439405989999159, "nonpadding_tokens": 7671, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4409427850000611, "nonpadding_tokens": 19451, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004032279999819366, "prefetch": {"data_wait_seconds": 0.003157788999942568, "consumed_batches": 316, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 321, "tokens": 29315890, "loss_tokens": 68859, "loss": 4.190345942433088, "lr": 0.00029999766873154896, "optimizer_lrs": [0.019999844582103264, 0.00029999766873154896], "gradient_norm": 1.8670389652252197, "seconds": 0.30627506500002255, "nonpadding_tokens_per_second": 224827.31331718084, "ranks": [{"compute_seconds": 0.3035542169999417, "nonpadding_tokens": 7899, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2990400060000411, "nonpadding_tokens": 8316, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30244801600008486, "nonpadding_tokens": 5092, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30394162999994023, "nonpadding_tokens": 8687, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30267490199992153, "nonpadding_tokens": 7521, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3038423050001029, "nonpadding_tokens": 11110, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.30274373700001433, "nonpadding_tokens": 15447, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30300574000000324, "nonpadding_tokens": 4787, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037017600004674023, "prefetch": {"data_wait_seconds": 0.0031673389999014034, "consumed_batches": 317, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 322, "tokens": 29416830, "loss_tokens": 100940, "loss": 4.514630597384585, "lr": 0.00029999761793824683, "optimizer_lrs": [0.019999841195883123, 0.00029999761793824683], "gradient_norm": 0.6938269138336182, "seconds": 0.36513617299999623, "nonpadding_tokens_per_second": 276444.8100845956, "ranks": [{"compute_seconds": 0.3613557820000324, "nonpadding_tokens": 14888, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.36234598099997584, "nonpadding_tokens": 7888, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3603033279999863, "nonpadding_tokens": 5934, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36249144500004604, "nonpadding_tokens": 18293, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36022473700006685, "nonpadding_tokens": 10702, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3585789090000162, "nonpadding_tokens": 13451, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3615373160000672, "nonpadding_tokens": 23275, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35919351799998367, "nonpadding_tokens": 6509, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004531429999587999, "prefetch": {"data_wait_seconds": 0.0031765259999474438, "consumed_batches": 318, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 323, "tokens": 29480592, "loss_tokens": 63762, "loss": 4.309045258147486, "lr": 0.00029999758557092593, "optimizer_lrs": [0.01999983903806173, 0.00029999758557092593], "gradient_norm": 0.7899003028869629, "seconds": 0.31292578800002957, "nonpadding_tokens_per_second": 203760.77154751457, "ranks": [{"compute_seconds": 0.3092701200000647, "nonpadding_tokens": 11947, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30512568899996495, "nonpadding_tokens": 5827, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30820140899993476, "nonpadding_tokens": 7553, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3092063330000201, "nonpadding_tokens": 2837, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30724539800007733, "nonpadding_tokens": 4815, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31046823000008317, "nonpadding_tokens": 8771, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3080528840000625, "nonpadding_tokens": 14996, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3060078439999643, "nonpadding_tokens": 7016, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004063810000616286, "prefetch": {"data_wait_seconds": 0.0031861409999009993, "consumed_batches": 319, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 324, "tokens": 29598716, "loss_tokens": 118124, "loss": 4.571563568792117, "lr": 0.0002999975250308481, "optimizer_lrs": [0.01999983500205654, 0.0002999975250308481], "gradient_norm": 0.7961740493774414, "seconds": 0.4038594539999849, "nonpadding_tokens_per_second": 292487.8910968973, "ranks": [{"compute_seconds": 0.3999067620000005, "nonpadding_tokens": 22168, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39591440300000613, "nonpadding_tokens": 11956, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3963316379999924, "nonpadding_tokens": 6663, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.39623644100004185, "nonpadding_tokens": 20237, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39788092699996014, "nonpadding_tokens": 15543, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4013051729999688, "nonpadding_tokens": 15740, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.39884925000001203, "nonpadding_tokens": 14140, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.39900142200008304, "nonpadding_tokens": 11677, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004312589999244665, "prefetch": {"data_wait_seconds": 0.0031969589998652737, "consumed_batches": 320, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 325, "tokens": 29673756, "loss_tokens": 75040, "loss": 4.4017998734008525, "lr": 0.0002999974861825134, "optimizer_lrs": [0.01999983241216756, 0.0002999974861825134], "gradient_norm": 0.7656188011169434, "seconds": 0.31788453900003333, "nonpadding_tokens_per_second": 236060.55279081105, "ranks": [{"compute_seconds": 0.27687477500001023, "nonpadding_tokens": 10250, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2822508540000399, "nonpadding_tokens": 7490, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2785607120000577, "nonpadding_tokens": 4817, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.27854387400009273, "nonpadding_tokens": 8770, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2791621829999258, "nonpadding_tokens": 12129, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.28347369899995556, "nonpadding_tokens": 13643, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.2815359549999812, "nonpadding_tokens": 5467, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.27898427700006323, "nonpadding_tokens": 12474, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0007374130000243895, "prefetch": {"data_wait_seconds": 0.0032072059998426994, "consumed_batches": 321, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 326, "tokens": 29740357, "loss_tokens": 66601, "loss": 4.078646529331391, "lr": 0.00029999745144966333, "optimizer_lrs": [0.019999830096644224, 0.00029999745144966333], "gradient_norm": 0.5396074652671814, "seconds": 0.4797475300000542, "nonpadding_tokens_per_second": 138825.10244501408, "ranks": [{"compute_seconds": 0.4731216679999761, "nonpadding_tokens": 14472, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.472366915000066, "nonpadding_tokens": 10012, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.47595217000002776, "nonpadding_tokens": 2365, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.47726966599998377, "nonpadding_tokens": 6709, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.47306614900003296, "nonpadding_tokens": 2387, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4727462740000874, "nonpadding_tokens": 10816, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.47531892699998934, "nonpadding_tokens": 8514, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.47615491700003076, "nonpadding_tokens": 11326, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003745190000472576, "prefetch": {"data_wait_seconds": 0.0032160469997961627, "consumed_batches": 322, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 327, "tokens": 29819015, "loss_tokens": 78658, "loss": 4.353731502199395, "lr": 0.00029999741012209203, "optimizer_lrs": [0.019999827341472803, 0.00029999741012209203], "gradient_norm": 0.5290199518203735, "seconds": 0.3425294330000952, "nonpadding_tokens_per_second": 229638.65998627376, "ranks": [{"compute_seconds": 0.3385851799999955, "nonpadding_tokens": 11615, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3347915470000089, "nonpadding_tokens": 1359, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.336486212000068, "nonpadding_tokens": 11101, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34016277699993225, "nonpadding_tokens": 17108, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.338437239999962, "nonpadding_tokens": 2749, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3398589839999886, "nonpadding_tokens": 3572, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3386095359999217, "nonpadding_tokens": 19557, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33833520899997893, "nonpadding_tokens": 11597, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003929289999859975, "prefetch": {"data_wait_seconds": 0.0032303409998348798, "consumed_batches": 323, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 328, "tokens": 29916968, "loss_tokens": 97953, "loss": 4.246756097311976, "lr": 0.0002999973581920671, "optimizer_lrs": [0.01999982387947114, 0.0002999973581920671], "gradient_norm": 0.5839954614639282, "seconds": 0.3933647630000223, "nonpadding_tokens_per_second": 249013.1532193046, "ranks": [{"compute_seconds": 0.39044431299998905, "nonpadding_tokens": 3253, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.38552697200009334, "nonpadding_tokens": 10358, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.38592236200008756, "nonpadding_tokens": 19232, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3863068050000038, "nonpadding_tokens": 8595, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3893208309999636, "nonpadding_tokens": 5121, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3909934490000069, "nonpadding_tokens": 9871, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.38612200299996857, "nonpadding_tokens": 28460, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.38922660399998676, "nonpadding_tokens": 13063, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003629830000591028, "prefetch": {"data_wait_seconds": 0.003241062999791211, "consumed_batches": 324, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 329, "tokens": 29991101, "loss_tokens": 74133, "loss": 4.562162768267843, "lr": 0.0002999973185475902, "optimizer_lrs": [0.019999821236506014, 0.0002999973185475902], "gradient_norm": 0.7028056383132935, "seconds": 0.3534540580000112, "nonpadding_tokens_per_second": 209738.71517977494, "ranks": [{"compute_seconds": 0.3490431909999643, "nonpadding_tokens": 23613, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35106203399993774, "nonpadding_tokens": 1739, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3469756150000194, "nonpadding_tokens": 5544, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3464851890000773, "nonpadding_tokens": 8439, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34825560299998415, "nonpadding_tokens": 13508, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.347060899999974, "nonpadding_tokens": 2795, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.34852244000001065, "nonpadding_tokens": 5448, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3499165670000366, "nonpadding_tokens": 13047, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039375599999402766, "prefetch": {"data_wait_seconds": 0.0032496679998530453, "consumed_batches": 325, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 330, "tokens": 30088211, "loss_tokens": 97110, "loss": 4.546671622386984, "lr": 0.0002999972661688907, "optimizer_lrs": [0.019999817744592713, 0.0002999972661688907], "gradient_norm": 0.6179990768432617, "seconds": 0.30895362899991596, "nonpadding_tokens_per_second": 314319.01387384714, "ranks": [{"compute_seconds": 0.3049244840000256, "nonpadding_tokens": 4045, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30654206000008344, "nonpadding_tokens": 6051, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3026182199999994, "nonpadding_tokens": 12816, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3064805270000761, "nonpadding_tokens": 13905, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3028296129999717, "nonpadding_tokens": 13134, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3023715299999594, "nonpadding_tokens": 16499, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.30539889600004244, "nonpadding_tokens": 11814, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3029131339999367, "nonpadding_tokens": 18846, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039497200009463995, "prefetch": {"data_wait_seconds": 0.0032596469998225075, "consumed_batches": 326, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 331, "tokens": 30217968, "loss_tokens": 129757, "loss": 4.3532482833296084, "lr": 0.00029999719539046974, "optimizer_lrs": [0.019999813026031318, 0.00029999719539046974], "gradient_norm": 0.772015392780304, "seconds": 0.37914142800002537, "nonpadding_tokens_per_second": 342239.04437051207, "ranks": [{"compute_seconds": 0.374644461999992, "nonpadding_tokens": 18569, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3710702539999602, "nonpadding_tokens": 25263, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3717435390000219, "nonpadding_tokens": 12176, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3756143320000547, "nonpadding_tokens": 18029, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3723807400000396, "nonpadding_tokens": 9793, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37666970800000854, "nonpadding_tokens": 10424, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.37534147799999573, "nonpadding_tokens": 11501, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3750327710000647, "nonpadding_tokens": 24002, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003973579999865251, "prefetch": {"data_wait_seconds": 0.003270211999733874, "consumed_batches": 327, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 332, "tokens": 30319922, "loss_tokens": 101954, "loss": 4.64012324185417, "lr": 0.00029999713914314284, "optimizer_lrs": [0.019999809276209524, 0.00029999713914314284], "gradient_norm": 1.0122767686843872, "seconds": 0.32832203400005255, "nonpadding_tokens_per_second": 310530.48361653264, "ranks": [{"compute_seconds": 0.3235361000000694, "nonpadding_tokens": 9898, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3259029369999098, "nonpadding_tokens": 11079, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3225460840000096, "nonpadding_tokens": 11413, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32147091699994235, "nonpadding_tokens": 11763, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3223133570000982, "nonpadding_tokens": 12642, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3220165169999518, "nonpadding_tokens": 10594, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32268734799993126, "nonpadding_tokens": 21971, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32464943099989796, "nonpadding_tokens": 12594, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004095259999985501, "prefetch": {"data_wait_seconds": 0.003280294999740363, "consumed_batches": 328, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 333, "tokens": 30406767, "loss_tokens": 86845, "loss": 4.4321607461569466, "lr": 0.00029999709079091497, "optimizer_lrs": [0.019999806052727664, 0.00029999709079091497], "gradient_norm": 0.6377284526824951, "seconds": 0.3537586200000078, "nonpadding_tokens_per_second": 245492.25118528018, "ranks": [{"compute_seconds": 0.3188208980000127, "nonpadding_tokens": 27022, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3268757169998935, "nonpadding_tokens": 7270, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3260499149999987, "nonpadding_tokens": 8777, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3270216150000351, "nonpadding_tokens": 5190, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32350125800007845, "nonpadding_tokens": 5133, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32320564700000887, "nonpadding_tokens": 2545, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32412874999999985, "nonpadding_tokens": 18358, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.323869893000051, "nonpadding_tokens": 12550, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0223796579999771, "prefetch": {"data_wait_seconds": 0.0032898839997415052, "consumed_batches": 329, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 334, "tokens": 30491030, "loss_tokens": 84263, "loss": 4.379761505643046, "lr": 0.00029999704348895775, "optimizer_lrs": [0.01999980289926385, 0.00029999704348895775], "gradient_norm": 0.5186340808868408, "seconds": 0.3669022640000321, "nonpadding_tokens_per_second": 229660.61610345534, "ranks": [{"compute_seconds": 0.3644721269999991, "nonpadding_tokens": 21236, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35986038999999437, "nonpadding_tokens": 19427, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3629509830000188, "nonpadding_tokens": 12758, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3630791230000341, "nonpadding_tokens": 11357, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36233509900000627, "nonpadding_tokens": 4873, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35989696299998286, "nonpadding_tokens": 3827, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.362970636, "nonpadding_tokens": 1415, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3602455980000059, "nonpadding_tokens": 9370, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037118299997018767, "prefetch": {"data_wait_seconds": 0.003300277999755963, "consumed_batches": 330, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 335, "tokens": 30578610, "loss_tokens": 87580, "loss": 4.48173170244348, "lr": 0.0002999969939206957, "optimizer_lrs": [0.01999979959471305, 0.0002999969939206957], "gradient_norm": 0.5950552225112915, "seconds": 0.3576025440000876, "nonpadding_tokens_per_second": 244908.77223730975, "ranks": [{"compute_seconds": 0.35274098100001083, "nonpadding_tokens": 7261, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34997944500003086, "nonpadding_tokens": 7752, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3530995510000139, "nonpadding_tokens": 9642, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3548159429999487, "nonpadding_tokens": 3155, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3507856249999577, "nonpadding_tokens": 4220, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35507098400000814, "nonpadding_tokens": 17061, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3535926079999854, "nonpadding_tokens": 22250, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35343947199999093, "nonpadding_tokens": 16239, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00042339699996318814, "prefetch": {"data_wait_seconds": 0.003312228999789113, "consumed_batches": 331, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 336, "tokens": 30653352, "loss_tokens": 74742, "loss": 4.248982332557331, "lr": 0.0002999969512925512, "optimizer_lrs": [0.019999796752836748, 0.0002999969512925512], "gradient_norm": 0.6331764459609985, "seconds": 0.318416731999946, "nonpadding_tokens_per_second": 234730.12718443663, "ranks": [{"compute_seconds": 0.3127746610000486, "nonpadding_tokens": 16273, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3157701589999533, "nonpadding_tokens": 3374, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31151631700004145, "nonpadding_tokens": 2605, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3161369189999732, "nonpadding_tokens": 14330, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3114239899999802, "nonpadding_tokens": 11953, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3120977270000367, "nonpadding_tokens": 6808, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3147379680000313, "nonpadding_tokens": 10738, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31486766900002294, "nonpadding_tokens": 8661, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035720599998967373, "prefetch": {"data_wait_seconds": 0.0033218649998616456, "consumed_batches": 332, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 337, "tokens": 30714359, "loss_tokens": 61007, "loss": 4.486559431704558, "lr": 0.0002999969162755287, "optimizer_lrs": [0.019999794418368584, 0.0002999969162755287], "gradient_norm": 0.7133104801177979, "seconds": 0.313591233000011, "nonpadding_tokens_per_second": 194543.06619597957, "ranks": [{"compute_seconds": 0.31041319600001316, "nonpadding_tokens": 11529, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3072159079999892, "nonpadding_tokens": 2901, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3105659330000208, "nonpadding_tokens": 7779, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3111071040000297, "nonpadding_tokens": 8624, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3076304279999249, "nonpadding_tokens": 6070, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3112615109999979, "nonpadding_tokens": 8247, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3108370240000795, "nonpadding_tokens": 4289, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3111007720000316, "nonpadding_tokens": 11568, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003504109999994398, "prefetch": {"data_wait_seconds": 0.0033315479998918818, "consumed_batches": 333, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 338, "tokens": 30797440, "loss_tokens": 83081, "loss": 4.083720254931934, "lr": 0.0002999968682668286, "optimizer_lrs": [0.019999791217788575, 0.0002999968682668286], "gradient_norm": 0.5446655750274658, "seconds": 0.3426753359999566, "nonpadding_tokens_per_second": 242448.14631190884, "ranks": [{"compute_seconds": 0.33559143000002223, "nonpadding_tokens": 5576, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33390014999997675, "nonpadding_tokens": 10645, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33427467100000285, "nonpadding_tokens": 17278, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3342220880000468, "nonpadding_tokens": 19571, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33643193299997165, "nonpadding_tokens": 7150, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34030420999999933, "nonpadding_tokens": 7806, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3351484840000012, "nonpadding_tokens": 5323, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3375145310000107, "nonpadding_tokens": 9732, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036164499999813415, "prefetch": {"data_wait_seconds": 0.003340929999922082, "consumed_batches": 334, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 339, "tokens": 30891977, "loss_tokens": 94537, "loss": 4.100009123411997, "lr": 0.0002999968131871827, "optimizer_lrs": [0.01999978754581218, 0.0002999968131871827], "gradient_norm": 0.5436533093452454, "seconds": 0.37813939700004084, "nonpadding_tokens_per_second": 250005.6877172991, "ranks": [{"compute_seconds": 0.37485500499997215, "nonpadding_tokens": 1735, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3700314929999422, "nonpadding_tokens": 8353, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.37257020900005955, "nonpadding_tokens": 1092, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3758437420000291, "nonpadding_tokens": 19023, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37440375699998185, "nonpadding_tokens": 24891, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37222297400001025, "nonpadding_tokens": 16280, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3719703799999934, "nonpadding_tokens": 13125, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3747410639999771, "nonpadding_tokens": 10038, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034406099996431294, "prefetch": {"data_wait_seconds": 0.0033497709998755454, "consumed_batches": 335, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 340, "tokens": 31005478, "loss_tokens": 113501, "loss": 4.184913018387503, "lr": 0.00029999674642435697, "optimizer_lrs": [0.019999783094957133, 0.00029999674642435697], "gradient_norm": 0.7971717715263367, "seconds": 0.35278921700000865, "nonpadding_tokens_per_second": 321724.6858199672, "ranks": [{"compute_seconds": 0.34963440500007437, "nonpadding_tokens": 14768, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3501383260000921, "nonpadding_tokens": 19043, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34625655199999983, "nonpadding_tokens": 18144, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34983515299995815, "nonpadding_tokens": 17084, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3487091549999377, "nonpadding_tokens": 17960, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3459525450000456, "nonpadding_tokens": 13167, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3491924679999556, "nonpadding_tokens": 10041, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34647106499994607, "nonpadding_tokens": 3294, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003884739999193698, "prefetch": {"data_wait_seconds": 0.0033592139998290804, "consumed_batches": 336, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 341, "tokens": 31093574, "loss_tokens": 88096, "loss": 4.654571788162913, "lr": 0.0002999966941280346, "optimizer_lrs": [0.01999977960853564, 0.0002999966941280346], "gradient_norm": 0.7053629159927368, "seconds": 0.38737238299995624, "nonpadding_tokens_per_second": 227419.41311807442, "ranks": [{"compute_seconds": 0.3841939279999451, "nonpadding_tokens": 5938, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3837396709999439, "nonpadding_tokens": 15694, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3811593159999802, "nonpadding_tokens": 10917, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3848922829999992, "nonpadding_tokens": 10456, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.38379048799993143, "nonpadding_tokens": 19068, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3849316069999986, "nonpadding_tokens": 5843, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3830153549999977, "nonpadding_tokens": 10710, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3837846939999281, "nonpadding_tokens": 9470, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036766400000942667, "prefetch": {"data_wait_seconds": 0.0033682449998195807, "consumed_batches": 337, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 342, "tokens": 31168174, "loss_tokens": 74600, "loss": 4.696518934316354, "lr": 0.0002999966495173021, "optimizer_lrs": [0.019999776634486807, 0.0002999966495173021], "gradient_norm": 0.6135555505752563, "seconds": 0.31151006699997197, "nonpadding_tokens_per_second": 239478.61691419018, "ranks": [{"compute_seconds": 0.3084737460000042, "nonpadding_tokens": 6196, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30519595700002355, "nonpadding_tokens": 10690, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3080963900000597, "nonpadding_tokens": 5147, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3091597489999458, "nonpadding_tokens": 9799, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3077910010000551, "nonpadding_tokens": 571, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30453407699997115, "nonpadding_tokens": 12112, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.30716659600000185, "nonpadding_tokens": 19683, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.307889247999924, "nonpadding_tokens": 10402, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003815219999978581, "prefetch": {"data_wait_seconds": 0.003377948999855107, "consumed_batches": 338, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 343, "tokens": 31276533, "loss_tokens": 108359, "loss": 4.5868917210384, "lr": 0.0002999965841861776, "optimizer_lrs": [0.019999772279078506, 0.0002999965841861776], "gradient_norm": 0.5735172033309937, "seconds": 0.4160315020000098, "nonpadding_tokens_per_second": 260458.64190351008, "ranks": [{"compute_seconds": 0.3621532339999476, "nonpadding_tokens": 21894, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3706467839999732, "nonpadding_tokens": 12177, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3702673909999703, "nonpadding_tokens": 8685, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3716692409999496, "nonpadding_tokens": 17529, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37036856100007753, "nonpadding_tokens": 5460, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3679153379999889, "nonpadding_tokens": 7005, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.37040423200005534, "nonpadding_tokens": 18506, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36791275300004145, "nonpadding_tokens": 17103, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.027158533000033458, "prefetch": {"data_wait_seconds": 0.0033877019998271862, "consumed_batches": 339, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 344, "tokens": 31387628, "loss_tokens": 111095, "loss": 4.360191165218957, "lr": 0.00029999651655058596, "optimizer_lrs": [0.019999767770039065, 0.00029999651655058596], "gradient_norm": 0.4996400773525238, "seconds": 0.3861150489999545, "nonpadding_tokens_per_second": 287725.1230889296, "ranks": [{"compute_seconds": 0.38026881799999046, "nonpadding_tokens": 16851, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3837417759999653, "nonpadding_tokens": 22624, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3785329869999714, "nonpadding_tokens": 6382, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38330403700001625, "nonpadding_tokens": 4594, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37905493200003093, "nonpadding_tokens": 15446, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3795627040000227, "nonpadding_tokens": 11323, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.38160908199995447, "nonpadding_tokens": 22557, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.38182806299994354, "nonpadding_tokens": 11318, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036112199995841365, "prefetch": {"data_wait_seconds": 0.0033969569998362203, "consumed_batches": 340, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 345, "tokens": 31506238, "loss_tokens": 118610, "loss": 4.456928694882388, "lr": 0.0002999964436079421, "optimizer_lrs": [0.019999762907196142, 0.0002999964436079421], "gradient_norm": 0.4600357413291931, "seconds": 0.3238358139999491, "nonpadding_tokens_per_second": 366265.8510031835, "ranks": [{"compute_seconds": 0.32150599800002055, "nonpadding_tokens": 9528, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3181456940000089, "nonpadding_tokens": 14960, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3183090390000416, "nonpadding_tokens": 11055, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31774400500000866, "nonpadding_tokens": 25342, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3208035459999792, "nonpadding_tokens": 19059, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31791418999989673, "nonpadding_tokens": 14044, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.32045249399993736, "nonpadding_tokens": 16747, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.321017396000002, "nonpadding_tokens": 7875, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039515399998890643, "prefetch": {"data_wait_seconds": 0.003406903999803035, "consumed_batches": 341, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 346, "tokens": 31600407, "loss_tokens": 94169, "loss": 4.253707430258365, "lr": 0.00029999638515776405, "optimizer_lrs": [0.019999759010517603, 0.00029999638515776405], "gradient_norm": 0.4998161494731903, "seconds": 0.31484411100007037, "nonpadding_tokens_per_second": 299097.2252931196, "ranks": [{"compute_seconds": 0.31044813100004376, "nonpadding_tokens": 4568, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3124525859999494, "nonpadding_tokens": 11308, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3115920489999553, "nonpadding_tokens": 10343, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3124229040000728, "nonpadding_tokens": 12949, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3088529330000256, "nonpadding_tokens": 17348, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3084137920000103, "nonpadding_tokens": 17833, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31134857300003205, "nonpadding_tokens": 10478, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31118114200000946, "nonpadding_tokens": 9342, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003867039999931876, "prefetch": {"data_wait_seconds": 0.003417608999825461, "consumed_batches": 342, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 347, "tokens": 31685055, "loss_tokens": 84648, "loss": 3.938968581655798, "lr": 0.0002999963322106262, "optimizer_lrs": [0.019999755480708415, 0.0002999963322106262], "gradient_norm": 0.7907413244247437, "seconds": 0.3409151649999558, "nonpadding_tokens_per_second": 248296.3760207352, "ranks": [{"compute_seconds": 0.33816295700000865, "nonpadding_tokens": 13650, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3385487990000229, "nonpadding_tokens": 4640, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33601220200000625, "nonpadding_tokens": 12606, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3348243030000049, "nonpadding_tokens": 7121, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.338140113999998, "nonpadding_tokens": 6286, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3356410280000546, "nonpadding_tokens": 10918, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.33852206900007786, "nonpadding_tokens": 11471, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3357860300000084, "nonpadding_tokens": 17956, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003892389999009538, "prefetch": {"data_wait_seconds": 0.003427565999913895, "consumed_batches": 343, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 348, "tokens": 31793630, "loss_tokens": 108575, "loss": 4.142592102233479, "lr": 0.0002999962637336665, "optimizer_lrs": [0.019999750915577767, 0.0002999962637336665], "gradient_norm": 0.3985871374607086, "seconds": 0.48900863799997296, "nonpadding_tokens_per_second": 222030.84273535063, "ranks": [{"compute_seconds": 0.4850465729999769, "nonpadding_tokens": 11723, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4647034619999886, "nonpadding_tokens": 15869, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.46098180199999206, "nonpadding_tokens": 8915, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.46252394399994046, "nonpadding_tokens": 7047, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.48359757499997613, "nonpadding_tokens": 21419, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.46067130300002646, "nonpadding_tokens": 15512, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4629048829999647, "nonpadding_tokens": 10757, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.46237233199997263, "nonpadding_tokens": 17333, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003610989999742742, "prefetch": {"data_wait_seconds": 0.0034363639999810403, "consumed_batches": 344, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 349, "tokens": 31896114, "loss_tokens": 102484, "loss": 4.481821552632606, "lr": 0.0002999961985172057, "optimizer_lrs": [0.01999974656781371, 0.0002999961985172057], "gradient_norm": 0.49669191241264343, "seconds": 0.29971313400005783, "nonpadding_tokens_per_second": 341940.30349027086, "ranks": [{"compute_seconds": 0.29275551099999575, "nonpadding_tokens": 9572, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.29631348600003093, "nonpadding_tokens": 19962, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.29262664999998833, "nonpadding_tokens": 13443, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.29139040799998384, "nonpadding_tokens": 23104, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2926754830000391, "nonpadding_tokens": 2220, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2970623590000514, "nonpadding_tokens": 13495, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.29522879900002863, "nonpadding_tokens": 9871, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2931129119999696, "nonpadding_tokens": 10817, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0006042750000005981, "prefetch": {"data_wait_seconds": 0.0034470119999241433, "consumed_batches": 345, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 350, "tokens": 31980420, "loss_tokens": 84306, "loss": 4.427958864137784, "lr": 0.0002999961444454471, "optimizer_lrs": [0.01999974296302981, 0.0002999961444454471], "gradient_norm": 0.4703199863433838, "seconds": 0.3242328879999832, "nonpadding_tokens_per_second": 260016.806191494, "ranks": [{"compute_seconds": 0.319807277000109, "nonpadding_tokens": 11039, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3184435040000153, "nonpadding_tokens": 18056, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31824091199996474, "nonpadding_tokens": 13415, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3178439119999439, "nonpadding_tokens": 8021, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3216891539999551, "nonpadding_tokens": 5923, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3184785070000089, "nonpadding_tokens": 12917, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.31952882500002033, "nonpadding_tokens": 664, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32177751899996565, "nonpadding_tokens": 14271, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039009800002531847, "prefetch": {"data_wait_seconds": 0.0034557479998511553, "consumed_batches": 346, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 351, "tokens": 32073285, "loss_tokens": 92865, "loss": 4.274736176169709, "lr": 0.00029999608444220596, "optimizer_lrs": [0.01999973896281373, 0.00029999608444220596], "gradient_norm": 0.8982901573181152, "seconds": 0.43853893400000743, "nonpadding_tokens_per_second": 211759.9893650456, "ranks": [{"compute_seconds": 0.43506287700006396, "nonpadding_tokens": 17159, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.432149458999902, "nonpadding_tokens": 8209, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.43351880099999107, "nonpadding_tokens": 11602, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.43620173700003306, "nonpadding_tokens": 13744, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4357519870000033, "nonpadding_tokens": 10562, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4324833440000475, "nonpadding_tokens": 14900, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.4358519059999253, "nonpadding_tokens": 8283, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.43469410900002003, "nonpadding_tokens": 8406, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036706899993532716, "prefetch": {"data_wait_seconds": 0.0034647889998495884, "consumed_batches": 347, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 352, "tokens": 32169945, "loss_tokens": 96660, "loss": 4.541264483757501, "lr": 0.00029999602149479495, "optimizer_lrs": [0.019999734766319666, 0.00029999602149479495], "gradient_norm": 0.6971648335456848, "seconds": 0.33475021000003835, "nonpadding_tokens_per_second": 288752.61945314065, "ranks": [{"compute_seconds": 0.32870885299996644, "nonpadding_tokens": 9486, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32791553899994597, "nonpadding_tokens": 8985, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32754489000001286, "nonpadding_tokens": 6547, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33136186100000486, "nonpadding_tokens": 12099, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.330475455999931, "nonpadding_tokens": 23804, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33230575100003534, "nonpadding_tokens": 21553, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.328972312000019, "nonpadding_tokens": 8226, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3301164739999649, "nonpadding_tokens": 5960, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004094460000487743, "prefetch": {"data_wait_seconds": 0.0034750129998428747, "consumed_batches": 348, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 353, "tokens": 32261238, "loss_tokens": 91293, "loss": 4.264304218286178, "lr": 0.0002999959615815958, "optimizer_lrs": [0.01999973077210639, 0.0002999959615815958], "gradient_norm": 0.4895753860473633, "seconds": 0.3566588920000413, "nonpadding_tokens_per_second": 255967.2618508259, "ranks": [{"compute_seconds": 0.351134696000031, "nonpadding_tokens": 4438, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3529320650000045, "nonpadding_tokens": 15562, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3530736980000029, "nonpadding_tokens": 19263, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3543212060000087, "nonpadding_tokens": 9103, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.351806095000029, "nonpadding_tokens": 6610, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3535620940000399, "nonpadding_tokens": 8901, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3509840010000289, "nonpadding_tokens": 16378, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3528851930000201, "nonpadding_tokens": 11038, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003512829999863243, "prefetch": {"data_wait_seconds": 0.003483815999743456, "consumed_batches": 349, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 354, "tokens": 32353670, "loss_tokens": 92432, "loss": 3.9996771275099534, "lr": 0.00029999590046473544, "optimizer_lrs": [0.01999972669764903, 0.00029999590046473544], "gradient_norm": 0.681440532207489, "seconds": 0.31898915800002214, "nonpadding_tokens_per_second": 289765.33428134123, "ranks": [{"compute_seconds": 0.31354146800003946, "nonpadding_tokens": 12438, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31665786700000353, "nonpadding_tokens": 16955, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31320544499999414, "nonpadding_tokens": 20211, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31225251000000753, "nonpadding_tokens": 15344, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31533947399998397, "nonpadding_tokens": 5909, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31307109700003366, "nonpadding_tokens": 5170, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19195232256}, {"compute_seconds": 0.3158615349999536, "nonpadding_tokens": 6112, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3130761989999655, "nonpadding_tokens": 10293, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037384100005510845, "prefetch": {"data_wait_seconds": 0.0034929899998132896, "consumed_batches": 350, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 355, "tokens": 32456470, "loss_tokens": 102800, "loss": 4.374459508754864, "lr": 0.00029999583195335316, "optimizer_lrs": [0.019999722130223545, 0.00029999583195335316], "gradient_norm": 0.5019140839576721, "seconds": 0.3630992180000021, "nonpadding_tokens_per_second": 283118.2082028042, "ranks": [{"compute_seconds": 0.35755152100000487, "nonpadding_tokens": 17744, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3594732470000963, "nonpadding_tokens": 1732, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3587603890000537, "nonpadding_tokens": 13218, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36066433500002404, "nonpadding_tokens": 3930, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3593523700000105, "nonpadding_tokens": 13071, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3566287310000007, "nonpadding_tokens": 9399, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.35940875200003575, "nonpadding_tokens": 17499, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3569740970000339, "nonpadding_tokens": 26207, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000454027999921891, "prefetch": {"data_wait_seconds": 0.0035003529997084115, "consumed_batches": 351, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 356, "tokens": 32558468, "loss_tokens": 101998, "loss": 4.261982158963901, "lr": 0.00029999576341535535, "optimizer_lrs": [0.019999717561023693, 0.00029999576341535535], "gradient_norm": 0.4832689166069031, "seconds": 0.4092869099999916, "nonpadding_tokens_per_second": 249209.04506817012, "ranks": [{"compute_seconds": 0.40584293799997795, "nonpadding_tokens": 19345, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40285924500005876, "nonpadding_tokens": 15045, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.40276427100002365, "nonpadding_tokens": 17460, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.402680657000019, "nonpadding_tokens": 3450, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4036132840000164, "nonpadding_tokens": 19198, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4069521290000466, "nonpadding_tokens": 6184, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.40509404300007645, "nonpadding_tokens": 12711, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.40467182899999443, "nonpadding_tokens": 8605, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003509250000206521, "prefetch": {"data_wait_seconds": 0.0035106329996779095, "consumed_batches": 352, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 357, "tokens": 32649499, "loss_tokens": 91031, "loss": 4.232340150608034, "lr": 0.00029999570177467955, "optimizer_lrs": [0.019999713451645303, 0.00029999570177467955], "gradient_norm": 1.616583228111267, "seconds": 0.35560696400000325, "nonpadding_tokens_per_second": 255987.675201994, "ranks": [{"compute_seconds": 0.3522857020000174, "nonpadding_tokens": 12742, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3528771989999768, "nonpadding_tokens": 20001, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3498877040000252, "nonpadding_tokens": 4433, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35325010299993664, "nonpadding_tokens": 16152, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3520262709999997, "nonpadding_tokens": 12772, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34828470299999026, "nonpadding_tokens": 11118, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3518963779999922, "nonpadding_tokens": 1731, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3516570420000562, "nonpadding_tokens": 12082, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038918099994589284, "prefetch": {"data_wait_seconds": 0.003519915999731893, "consumed_batches": 353, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 358, "tokens": 32742831, "loss_tokens": 93332, "loss": 4.1948708642266315, "lr": 0.00029999563811370023, "optimizer_lrs": [0.019999709207580017, 0.00029999563811370023], "gradient_norm": 0.9822529554367065, "seconds": 0.34390619999999217, "nonpadding_tokens_per_second": 271387.9540409627, "ranks": [{"compute_seconds": 0.338958367000032, "nonpadding_tokens": 14453, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34087790700004916, "nonpadding_tokens": 20864, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34026551099998414, "nonpadding_tokens": 13343, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3415395920000037, "nonpadding_tokens": 17332, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3404743649999773, "nonpadding_tokens": 9821, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33724882399997114, "nonpadding_tokens": 3928, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3379572349999762, "nonpadding_tokens": 6909, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3402089190000197, "nonpadding_tokens": 6682, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003944449999835342, "prefetch": {"data_wait_seconds": 0.0035301869997965696, "consumed_batches": 354, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 359, "tokens": 32824057, "loss_tokens": 81226, "loss": 4.288525610641913, "lr": 0.00029999558232926234, "optimizer_lrs": [0.01999970548861749, 0.00029999558232926234], "gradient_norm": 0.5719711184501648, "seconds": 0.28155304000006254, "nonpadding_tokens_per_second": 288492.71171066724, "ranks": [{"compute_seconds": 0.27498034099994584, "nonpadding_tokens": 9903, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2792301419999603, "nonpadding_tokens": 6867, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2784226309998985, "nonpadding_tokens": 9163, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.27896136799995475, "nonpadding_tokens": 13949, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2765923870000506, "nonpadding_tokens": 11814, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2747545239999454, "nonpadding_tokens": 11378, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2763121890000093, "nonpadding_tokens": 6670, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2776138650000348, "nonpadding_tokens": 11482, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003662689999828217, "prefetch": {"data_wait_seconds": 0.0035386099997367637, "consumed_batches": 355, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 360, "tokens": 32922956, "loss_tokens": 98899, "loss": 4.72311398497457, "lr": 0.0002999955139288356, "optimizer_lrs": [0.01999970092858904, 0.0002999955139288356], "gradient_norm": 0.6526896953582764, "seconds": 0.36194690499996796, "nonpadding_tokens_per_second": 273241.7341709518, "ranks": [{"compute_seconds": 0.35797235200004707, "nonpadding_tokens": 18207, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3596064490000117, "nonpadding_tokens": 8116, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35599318600009155, "nonpadding_tokens": 7817, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35497897099992315, "nonpadding_tokens": 14752, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3584875359999842, "nonpadding_tokens": 6783, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.359579558000064, "nonpadding_tokens": 15778, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3577488160000257, "nonpadding_tokens": 4815, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3583053160000418, "nonpadding_tokens": 22631, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000353227000005063, "prefetch": {"data_wait_seconds": 0.00354870399974061, "consumed_batches": 356, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 361, "tokens": 33033774, "loss_tokens": 110818, "loss": 4.46737048584165, "lr": 0.00029999543666072933, "optimizer_lrs": [0.019999695777381955, 0.00029999543666072933], "gradient_norm": 0.553100049495697, "seconds": 0.38572646000000077, "nonpadding_tokens_per_second": 287296.8579858374, "ranks": [{"compute_seconds": 0.38307389199997033, "nonpadding_tokens": 9700, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.38340395599993826, "nonpadding_tokens": 7594, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3821829550000757, "nonpadding_tokens": 21288, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3832404290000113, "nonpadding_tokens": 17545, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.38000281100005395, "nonpadding_tokens": 15977, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37941522500000247, "nonpadding_tokens": 3967, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.38224249400002464, "nonpadding_tokens": 13115, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.38229611999997815, "nonpadding_tokens": 21632, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003823810000085359, "prefetch": {"data_wait_seconds": 0.003558665999776167, "consumed_batches": 357, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 362, "tokens": 33115576, "loss_tokens": 81802, "loss": 4.236697299577028, "lr": 0.0002999953792008475, "optimizer_lrs": [0.01999969194672317, 0.0002999953792008475], "gradient_norm": 0.5133772492408752, "seconds": 0.34112020800000664, "nonpadding_tokens_per_second": 239804.0282620794, "ranks": [{"compute_seconds": 0.33656918199994834, "nonpadding_tokens": 17381, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33844172500005243, "nonpadding_tokens": 9446, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3360306629999741, "nonpadding_tokens": 14803, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3387560950000079, "nonpadding_tokens": 9449, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33657469199999923, "nonpadding_tokens": 8732, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33463096299999506, "nonpadding_tokens": 9563, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.33744509399991784, "nonpadding_tokens": 2312, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3351536400000441, "nonpadding_tokens": 10116, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004046049999715251, "prefetch": {"data_wait_seconds": 0.0035701179998568477, "consumed_batches": 358, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 363, "tokens": 33177922, "loss_tokens": 62346, "loss": 4.212069539344946, "lr": 0.0002999953351659676, "optimizer_lrs": [0.019999689011064508, 0.0002999953351659676], "gradient_norm": 0.5125063061714172, "seconds": 0.40817074599999614, "nonpadding_tokens_per_second": 152744.90053728787, "ranks": [{"compute_seconds": 0.4025927000000138, "nonpadding_tokens": 6259, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40016187699995953, "nonpadding_tokens": 10021, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4009389560000045, "nonpadding_tokens": 5932, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.403573978000054, "nonpadding_tokens": 13882, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4004449450000038, "nonpadding_tokens": 4746, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4057160139999496, "nonpadding_tokens": 3802, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4013184089999413, "nonpadding_tokens": 10815, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.40380813599995236, "nonpadding_tokens": 6889, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039340000000720465, "prefetch": {"data_wait_seconds": 0.0035797809997575314, "consumed_batches": 359, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 364, "tokens": 33241378, "loss_tokens": 63456, "loss": 4.303585850825769, "lr": 0.0002999952901326642, "optimizer_lrs": [0.019999686008844282, 0.0002999952901326642], "gradient_norm": 0.4972452223300934, "seconds": 1.1797151300000905, "nonpadding_tokens_per_second": 53789.25673352611, "ranks": [{"compute_seconds": 1.1766002489999892, "nonpadding_tokens": 5985, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 1.1773047950000546, "nonpadding_tokens": 7607, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 1.1731258199999957, "nonpadding_tokens": 4804, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 1.172385550999934, "nonpadding_tokens": 5120, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 1.1742051369999444, "nonpadding_tokens": 12248, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 1.1730063869999867, "nonpadding_tokens": 11961, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 1.1737244530000908, "nonpadding_tokens": 12601, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 1.1763126430000739, "nonpadding_tokens": 3130, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003987900000765876, "prefetch": {"data_wait_seconds": 0.0035901529996635873, "consumed_batches": 360, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 365, "tokens": 33332907, "loss_tokens": 91529, "loss": 4.3597340050694315, "lr": 0.00029999522479553435, "optimizer_lrs": [0.019999681653035624, 0.00029999522479553435], "gradient_norm": 0.5684394836425781, "seconds": 0.2515108909999526, "nonpadding_tokens_per_second": 363916.6464565435, "ranks": [{"compute_seconds": 0.24886591699998917, "nonpadding_tokens": 5904, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.24866519800002607, "nonpadding_tokens": 2052, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.24462942399998155, "nonpadding_tokens": 14708, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2491989460000923, "nonpadding_tokens": 6724, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.24561187999995582, "nonpadding_tokens": 17863, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.24477688900003614, "nonpadding_tokens": 16461, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2478637169999729, "nonpadding_tokens": 11707, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.24754241000005095, "nonpadding_tokens": 16110, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035253399994417123, "prefetch": {"data_wait_seconds": 0.003599285999598578, "consumed_batches": 361, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 366, "tokens": 33390762, "loss_tokens": 57855, "loss": 4.272660908737361, "lr": 0.00029999518326412474, "optimizer_lrs": [0.019999678884274983, 0.00029999518326412474], "gradient_norm": 0.5571070313453674, "seconds": 0.27238690299998325, "nonpadding_tokens_per_second": 212400.08004350914, "ranks": [{"compute_seconds": 0.26965021300009084, "nonpadding_tokens": 15132, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.27007459399999334, "nonpadding_tokens": 1652, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2691118750000214, "nonpadding_tokens": 8914, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2664501399999608, "nonpadding_tokens": 9470, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2690816670000231, "nonpadding_tokens": 10255, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2659952799999701, "nonpadding_tokens": 3547, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.26760090299990225, "nonpadding_tokens": 3144, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.26761388399995667, "nonpadding_tokens": 5741, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004041419999794016, "prefetch": {"data_wait_seconds": 0.0036097379995680967, "consumed_batches": 362, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 367, "tokens": 33491080, "loss_tokens": 100318, "loss": 4.549526256504317, "lr": 0.0002999951108242788, "optimizer_lrs": [0.01999967405495192, 0.0002999951108242788], "gradient_norm": 0.5607660412788391, "seconds": 0.327213087999894, "nonpadding_tokens_per_second": 306583.0911996787, "ranks": [{"compute_seconds": 0.32144728499997655, "nonpadding_tokens": 14889, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32489645599991945, "nonpadding_tokens": 8653, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32300397699998484, "nonpadding_tokens": 19794, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32007487600003515, "nonpadding_tokens": 19789, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32305308100001184, "nonpadding_tokens": 2511, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32069725600001675, "nonpadding_tokens": 5973, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3205826559999423, "nonpadding_tokens": 16055, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32355021199998646, "nonpadding_tokens": 12654, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037440699998114724, "prefetch": {"data_wait_seconds": 0.0036189599994713717, "consumed_batches": 363, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 368, "tokens": 33597404, "loss_tokens": 106324, "loss": 4.301667544486663, "lr": 0.0002999950334573146, "optimizer_lrs": [0.019999668897154307, 0.0002999950334573146], "gradient_norm": 0.5257465243339539, "seconds": 0.24884095100003378, "nonpadding_tokens_per_second": 427276.939638386, "ranks": [{"compute_seconds": 0.2444433329999356, "nonpadding_tokens": 18293, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.24469651800006886, "nonpadding_tokens": 7295, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.24125153399995725, "nonpadding_tokens": 17122, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.24527475700006107, "nonpadding_tokens": 9352, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.24232164900001862, "nonpadding_tokens": 11448, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2463843430000452, "nonpadding_tokens": 22643, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.24190190500007702, "nonpadding_tokens": 2213, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.24227787400002398, "nonpadding_tokens": 17958, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003870790000064517, "prefetch": {"data_wait_seconds": 0.0036279969995121064, "consumed_batches": 364, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 369, "tokens": 33714825, "loss_tokens": 117421, "loss": 3.996552735456179, "lr": 0.0002999949473098738, "optimizer_lrs": [0.019999663153991588, 0.0002999949473098738], "gradient_norm": 0.8771652579307556, "seconds": 0.32381128700001227, "nonpadding_tokens_per_second": 362621.7019420807, "ranks": [{"compute_seconds": 0.32110504599995693, "nonpadding_tokens": 6675, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3047066110000287, "nonpadding_tokens": 27424, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31924082500006534, "nonpadding_tokens": 10116, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30263429199999337, "nonpadding_tokens": 20695, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3031261450000784, "nonpadding_tokens": 12587, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3080302550000624, "nonpadding_tokens": 11151, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3033704399999806, "nonpadding_tokens": 18131, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3035628850000194, "nonpadding_tokens": 10642, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000376516999949672, "prefetch": {"data_wait_seconds": 0.0036378769995053517, "consumed_batches": 365, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 370, "tokens": 33848106, "loss_tokens": 133281, "loss": 4.001717705449389, "lr": 0.0002999948486289965, "optimizer_lrs": [0.019999656575266433, 0.0002999948486289965], "gradient_norm": 0.8632709383964539, "seconds": 0.34856025299995963, "nonpadding_tokens_per_second": 382375.78396529175, "ranks": [{"compute_seconds": 0.2848641369999996, "nonpadding_tokens": 24786, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3018662480000103, "nonpadding_tokens": 11152, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3011323819999916, "nonpadding_tokens": 23629, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3027926329999673, "nonpadding_tokens": 18219, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.300718546999974, "nonpadding_tokens": 17578, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.29832478399998763, "nonpadding_tokens": 12250, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.30135751400007393, "nonpadding_tokens": 12965, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30014624800003276, "nonpadding_tokens": 12702, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.024989613000002464, "prefetch": {"data_wait_seconds": 0.003647686999443067, "consumed_batches": 366, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 371, "tokens": 33959968, "loss_tokens": 111862, "loss": 3.888960281418176, "lr": 0.00029999476507008876, "optimizer_lrs": [0.019999651004672585, 0.00029999476507008876], "gradient_norm": 0.8571313619613647, "seconds": 0.30718496999998024, "nonpadding_tokens_per_second": 364151.93100107467, "ranks": [{"compute_seconds": 0.30415283499996804, "nonpadding_tokens": 9480, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.301689877000058, "nonpadding_tokens": 2079, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30378670500010685, "nonpadding_tokens": 12232, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3013434079999797, "nonpadding_tokens": 19242, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3047100889999683, "nonpadding_tokens": 18402, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30198016800000005, "nonpadding_tokens": 14004, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.30427616399992985, "nonpadding_tokens": 18925, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30232354800000394, "nonpadding_tokens": 17498, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0005571140000029118, "prefetch": {"data_wait_seconds": 0.0036586729994496636, "consumed_batches": 367, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 372, "tokens": 34069620, "loss_tokens": 109652, "loss": 3.998677064713822, "lr": 0.00029999468250956366, "optimizer_lrs": [0.01999964550063758, 0.00029999468250956366], "gradient_norm": 0.727311372756958, "seconds": 0.36017656500007433, "nonpadding_tokens_per_second": 304439.57396277954, "ranks": [{"compute_seconds": 0.35607560999994803, "nonpadding_tokens": 11559, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3577873860000409, "nonpadding_tokens": 19730, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35400660599998446, "nonpadding_tokens": 12364, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35347608100005345, "nonpadding_tokens": 17431, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3561308810000128, "nonpadding_tokens": 21492, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35350554299998294, "nonpadding_tokens": 5000, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.356345092999959, "nonpadding_tokens": 14551, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35672201199997744, "nonpadding_tokens": 7525, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003988319999734813, "prefetch": {"data_wait_seconds": 0.0036689079994403073, "consumed_batches": 368, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 373, "tokens": 34143287, "loss_tokens": 73667, "loss": 4.155576360514206, "lr": 0.0002999946266805543, "optimizer_lrs": [0.01999964177870362, 0.0002999946266805543], "gradient_norm": 0.5564354658126831, "seconds": 0.3460600920000161, "nonpadding_tokens_per_second": 212873.4335538366, "ranks": [{"compute_seconds": 0.3429247970000233, "nonpadding_tokens": 10642, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3411887240000624, "nonpadding_tokens": 9297, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34136421400000927, "nonpadding_tokens": 9508, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34081885600005535, "nonpadding_tokens": 17541, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3437572180000643, "nonpadding_tokens": 15996, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3410431530001006, "nonpadding_tokens": 1425, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.34207421300004626, "nonpadding_tokens": 7469, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3437326619999794, "nonpadding_tokens": 1789, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00041239500001211127, "prefetch": {"data_wait_seconds": 0.00367927999946005, "consumed_batches": 369, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 374, "tokens": 34207483, "loss_tokens": 64196, "loss": 4.324244306498847, "lr": 0.0002999945777914735, "optimizer_lrs": [0.019999638519431567, 0.0002999945777914735], "gradient_norm": 0.6162387728691101, "seconds": 0.3251882790000309, "nonpadding_tokens_per_second": 197411.7892483877, "ranks": [{"compute_seconds": 0.3198210719999679, "nonpadding_tokens": 13430, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3216759640000646, "nonpadding_tokens": 7169, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31827865399998245, "nonpadding_tokens": 12950, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32214519899991956, "nonpadding_tokens": 2497, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3199588830000266, "nonpadding_tokens": 2799, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3228394269999626, "nonpadding_tokens": 2575, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3188729339999554, "nonpadding_tokens": 14453, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32083014700003787, "nonpadding_tokens": 8323, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035900500006391667, "prefetch": {"data_wait_seconds": 0.0036887959995510755, "consumed_batches": 370, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 375, "tokens": 34271995, "loss_tokens": 64512, "loss": 4.195533873542907, "lr": 0.0002999945284387047, "optimizer_lrs": [0.01999963522924698, 0.0002999945284387047], "gradient_norm": 0.693874716758728, "seconds": 0.475203669999928, "nonpadding_tokens_per_second": 135756.52730125122, "ranks": [{"compute_seconds": 0.4701885780000339, "nonpadding_tokens": 4599, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.47194849400000294, "nonpadding_tokens": 7305, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.46839430200009247, "nonpadding_tokens": 11175, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4721304519999876, "nonpadding_tokens": 15237, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.47035321400005614, "nonpadding_tokens": 7746, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.47272186399993643, "nonpadding_tokens": 2578, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4707738539999582, "nonpadding_tokens": 7782, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.47069162699995104, "nonpadding_tokens": 8090, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003795229999923322, "prefetch": {"data_wait_seconds": 0.003699045999496775, "consumed_batches": 371, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 376, "tokens": 34351956, "loss_tokens": 79961, "loss": 4.546289753754955, "lr": 0.00029999446695688276, "optimizer_lrs": [0.019999631130458852, 0.00029999446695688276], "gradient_norm": 0.5889379382133484, "seconds": 0.31874419899997974, "nonpadding_tokens_per_second": 250862.6047183531, "ranks": [{"compute_seconds": 0.20029238099994018, "nonpadding_tokens": 12005, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.250503763000097, "nonpadding_tokens": 9141, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.25405437300003086, "nonpadding_tokens": 17434, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2502588980000837, "nonpadding_tokens": 3698, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.25358889000006, "nonpadding_tokens": 5653, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2546596749999708, "nonpadding_tokens": 12988, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.25371776299994053, "nonpadding_tokens": 14577, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.25327695399994354, "nonpadding_tokens": 4465, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.06218159800005196, "prefetch": {"data_wait_seconds": 0.003708206999590402, "consumed_batches": 372, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 377, "tokens": 34418295, "loss_tokens": 66339, "loss": 4.136842581287025, "lr": 0.0002999944156882845, "optimizer_lrs": [0.0199996277125523, 0.0002999944156882845], "gradient_norm": 0.6252909898757935, "seconds": 0.27859321000005366, "nonpadding_tokens_per_second": 238121.38135020313, "ranks": [{"compute_seconds": 0.27555141600009847, "nonpadding_tokens": 2329, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.27564056899996103, "nonpadding_tokens": 6921, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.27543791699997655, "nonpadding_tokens": 9221, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.27623261000007915, "nonpadding_tokens": 10997, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.27466341599995303, "nonpadding_tokens": 14889, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.27596398900004715, "nonpadding_tokens": 9769, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2748503200000414, "nonpadding_tokens": 7025, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2748699229999829, "nonpadding_tokens": 5188, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003879439999536771, "prefetch": {"data_wait_seconds": 0.0037175029995069053, "consumed_batches": 373, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 378, "tokens": 34495965, "loss_tokens": 77670, "loss": 4.324587195828506, "lr": 0.0002999943553623304, "optimizer_lrs": [0.01999962369082203, 0.0002999943553623304], "gradient_norm": 0.6426637768745422, "seconds": 0.6604211810000606, "nonpadding_tokens_per_second": 117606.76706701942, "ranks": [{"compute_seconds": 0.6560487199999443, "nonpadding_tokens": 16507, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.6528655910000225, "nonpadding_tokens": 10132, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.6569138489999204, "nonpadding_tokens": 5376, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.6577092449999782, "nonpadding_tokens": 21464, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.6574759829999266, "nonpadding_tokens": 6883, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.653758186999994, "nonpadding_tokens": 4483, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.6566508980000663, "nonpadding_tokens": 3818, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.6535305450000806, "nonpadding_tokens": 9007, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038270599998213584, "prefetch": {"data_wait_seconds": 0.003728038999497585, "consumed_batches": 374, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 379, "tokens": 34596547, "loss_tokens": 100582, "loss": 4.320988409954067, "lr": 0.00029999427675913047, "optimizer_lrs": [0.0199996184506087, 0.00029999427675913047], "gradient_norm": 0.8289785981178284, "seconds": 0.2608622569999852, "nonpadding_tokens_per_second": 385575.13515650405, "ranks": [{"compute_seconds": 0.2570515480000495, "nonpadding_tokens": 8284, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2583948999999848, "nonpadding_tokens": 14284, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2548270229999616, "nonpadding_tokens": 19051, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2534036949999745, "nonpadding_tokens": 11360, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2547808310000619, "nonpadding_tokens": 16980, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2540303370000174, "nonpadding_tokens": 11689, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.25458307499991406, "nonpadding_tokens": 9466, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.25706511999999293, "nonpadding_tokens": 9468, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00041147100000671344, "prefetch": {"data_wait_seconds": 0.0037377379994723015, "consumed_batches": 375, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 380, "tokens": 34740947, "loss_tokens": 144400, "loss": 4.150699013157895, "lr": 0.00029999416296264705, "optimizer_lrs": [0.01999961086417647, 0.00029999416296264705], "gradient_norm": 1.036263346672058, "seconds": 0.3792470220000723, "nonpadding_tokens_per_second": 380754.4730040687, "ranks": [{"compute_seconds": 0.3749688990000095, "nonpadding_tokens": 13588, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.37284021100003883, "nonpadding_tokens": 21287, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3691510290000224, "nonpadding_tokens": 17853, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37281903899997815, "nonpadding_tokens": 22546, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3724796190000461, "nonpadding_tokens": 29524, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3767762009999842, "nonpadding_tokens": 9147, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3727904559999615, "nonpadding_tokens": 18594, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37158826799998224, "nonpadding_tokens": 11861, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003996489999735786, "prefetch": {"data_wait_seconds": 0.0037471329994787084, "consumed_batches": 376, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 381, "tokens": 34834279, "loss_tokens": 93332, "loss": 4.853831617237389, "lr": 0.0002999940888150335, "optimizer_lrs": [0.019999605921002233, 0.0002999940888150335], "gradient_norm": 0.7280498743057251, "seconds": 0.3348428499999727, "nonpadding_tokens_per_second": 278733.7403202953, "ranks": [{"compute_seconds": 0.33126009399995837, "nonpadding_tokens": 9735, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32835085200008507, "nonpadding_tokens": 21949, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32867347299998073, "nonpadding_tokens": 12161, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33245536499998707, "nonpadding_tokens": 15420, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32950029899996025, "nonpadding_tokens": 9988, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3285811670000385, "nonpadding_tokens": 8873, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.32954419999998663, "nonpadding_tokens": 3124, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33101195100005043, "nonpadding_tokens": 12082, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003859709999005645, "prefetch": {"data_wait_seconds": 0.0037576079994323663, "consumed_batches": 377, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 382, "tokens": 34935585, "loss_tokens": 101306, "loss": 4.327897656604742, "lr": 0.00029999400780282554, "optimizer_lrs": [0.01999960052018837, 0.00029999400780282554], "gradient_norm": 0.5222294926643372, "seconds": 0.4068291849999923, "nonpadding_tokens_per_second": 249013.60014277714, "ranks": [{"compute_seconds": 0.4044266590000234, "nonpadding_tokens": 15414, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3947335350000003, "nonpadding_tokens": 10426, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3990989119999995, "nonpadding_tokens": 9823, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.395999364999966, "nonpadding_tokens": 6458, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4030599399999346, "nonpadding_tokens": 24918, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.39547880899999655, "nonpadding_tokens": 10404, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.39860317000000123, "nonpadding_tokens": 12300, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.39728947500009326, "nonpadding_tokens": 11563, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003874389999509731, "prefetch": {"data_wait_seconds": 0.0037669779994757846, "consumed_batches": 378, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 383, "tokens": 34988981, "loss_tokens": 53396, "loss": 4.1089302101281, "lr": 0.00029999396488131783, "optimizer_lrs": [0.019999597658754524, 0.00029999396488131783], "gradient_norm": 0.5697652101516724, "seconds": 0.3061111260000189, "nonpadding_tokens_per_second": 174433.38534515307, "ranks": [{"compute_seconds": 0.2847846939999954, "nonpadding_tokens": 10514, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2886459300000297, "nonpadding_tokens": 8031, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2857718529999147, "nonpadding_tokens": 8770, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.28975150400003713, "nonpadding_tokens": 4920, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.28793739099990034, "nonpadding_tokens": 3597, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2845317950000208, "nonpadding_tokens": 6133, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2880417999999736, "nonpadding_tokens": 7004, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2873838530000512, "nonpadding_tokens": 4427, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00041199200006758474, "prefetch": {"data_wait_seconds": 0.00377679299947431, "consumed_batches": 379, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 384, "tokens": 35086654, "loss_tokens": 97673, "loss": 4.325378431091499, "lr": 0.0002999938859721158, "optimizer_lrs": [0.019999592398141055, 0.0002999938859721158], "gradient_norm": 0.7128401398658752, "seconds": 0.3423886799999991, "nonpadding_tokens_per_second": 285269.3611249071, "ranks": [{"compute_seconds": 0.3390327789999219, "nonpadding_tokens": 5365, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33995622300005834, "nonpadding_tokens": 8947, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3389982289999125, "nonpadding_tokens": 14926, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3398329340000146, "nonpadding_tokens": 18413, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33693117199993594, "nonpadding_tokens": 14207, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3359080790000917, "nonpadding_tokens": 10295, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3389093089999733, "nonpadding_tokens": 19644, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33885683200003314, "nonpadding_tokens": 5876, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038765400006468553, "prefetch": {"data_wait_seconds": 0.0037855829995123713, "consumed_batches": 380, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 385, "tokens": 35160598, "loss_tokens": 73944, "loss": 3.646393723628692, "lr": 0.0002999938258925053, "optimizer_lrs": [0.019999588392833688, 0.0002999938258925053], "gradient_norm": 0.5036239624023438, "seconds": 0.3686196370000516, "nonpadding_tokens_per_second": 200597.01811271015, "ranks": [{"compute_seconds": 0.3617797550000432, "nonpadding_tokens": 12026, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3610538439999118, "nonpadding_tokens": 6535, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3609529670000029, "nonpadding_tokens": 16954, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36120402799997464, "nonpadding_tokens": 4701, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36135208899997906, "nonpadding_tokens": 12342, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3661561060000622, "nonpadding_tokens": 2872, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3635796639999853, "nonpadding_tokens": 3805, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3620767810000416, "nonpadding_tokens": 14709, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003791159999764204, "prefetch": {"data_wait_seconds": 0.003794184999492245, "consumed_batches": 381, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 386, "tokens": 35263664, "loss_tokens": 103066, "loss": 4.392627054508761, "lr": 0.0002999937416611787, "optimizer_lrs": [0.019999582777411914, 0.0002999937416611787], "gradient_norm": 0.5843669176101685, "seconds": 0.33993894299999283, "nonpadding_tokens_per_second": 303189.74075295095, "ranks": [{"compute_seconds": 0.33497554600000967, "nonpadding_tokens": 22010, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3375579710000238, "nonpadding_tokens": 16099, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33354926100003013, "nonpadding_tokens": 14877, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3329731339999853, "nonpadding_tokens": 3860, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33386455700008355, "nonpadding_tokens": 11803, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3334762920000003, "nonpadding_tokens": 19755, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3362568750000037, "nonpadding_tokens": 7692, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33627400399996077, "nonpadding_tokens": 6970, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035862999993696576, "prefetch": {"data_wait_seconds": 0.003803901999390291, "consumed_batches": 382, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 387, "tokens": 35331126, "loss_tokens": 67462, "loss": 4.2595595483383235, "lr": 0.00029999368621842646, "optimizer_lrs": [0.01999957908122843, 0.00029999368621842646], "gradient_norm": 0.5734723210334778, "seconds": 0.23618002800003524, "nonpadding_tokens_per_second": 285638.0387929751, "ranks": [{"compute_seconds": 0.2319000469999537, "nonpadding_tokens": 4849, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.23388960600004793, "nonpadding_tokens": 11039, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.23287314200001674, "nonpadding_tokens": 3690, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.22902606200000264, "nonpadding_tokens": 18059, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.23259339000003365, "nonpadding_tokens": 11401, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.23036858200009647, "nonpadding_tokens": 1725, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.23291783399997712, "nonpadding_tokens": 11300, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2328387400000338, "nonpadding_tokens": 5399, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035148500001014327, "prefetch": {"data_wait_seconds": 0.0038130889994363315, "consumed_batches": 383, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 388, "tokens": 35417595, "loss_tokens": 86469, "loss": 4.171050463171772, "lr": 0.000299993614797494, "optimizer_lrs": [0.019999574319832934, 0.000299993614797494], "gradient_norm": 0.5039421916007996, "seconds": 0.3477778589999616, "nonpadding_tokens_per_second": 248632.8492809818, "ranks": [{"compute_seconds": 0.34449388899997757, "nonpadding_tokens": 11476, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3444646069999635, "nonpadding_tokens": 7467, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34326275800003714, "nonpadding_tokens": 19411, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3445811809999668, "nonpadding_tokens": 17293, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3434309970000413, "nonpadding_tokens": 3264, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34537253399992096, "nonpadding_tokens": 12670, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3411330479999606, "nonpadding_tokens": 12976, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34320202699996116, "nonpadding_tokens": 1912, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004136740000149075, "prefetch": {"data_wait_seconds": 0.003823218999400524, "consumed_batches": 384, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 389, "tokens": 35488969, "loss_tokens": 71374, "loss": 4.099294298343935, "lr": 0.00029999355554198733, "optimizer_lrs": [0.019999570369465824, 0.00029999355554198733], "gradient_norm": 0.6423544883728027, "seconds": 0.3355968539999594, "nonpadding_tokens_per_second": 212677.7982251545, "ranks": [{"compute_seconds": 0.33315418899996985, "nonpadding_tokens": 7340, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3323474389999319, "nonpadding_tokens": 10517, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3305774929999643, "nonpadding_tokens": 5031, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33259616400005143, "nonpadding_tokens": 3146, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.329160611999896, "nonpadding_tokens": 6278, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32886508499996125, "nonpadding_tokens": 11218, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.33173108400001183, "nonpadding_tokens": 12420, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33166010599995843, "nonpadding_tokens": 15424, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003947180000523076, "prefetch": {"data_wait_seconds": 0.0038330139993831835, "consumed_batches": 385, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 390, "tokens": 35545673, "loss_tokens": 56704, "loss": 4.33892670711061, "lr": 0.0002999935082706017, "optimizer_lrs": [0.019999567218040113, 0.0002999935082706017], "gradient_norm": 0.5816837549209595, "seconds": 0.33031705399992006, "nonpadding_tokens_per_second": 171665.37214277082, "ranks": [{"compute_seconds": 0.326264740000056, "nonpadding_tokens": 1907, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3278863169999795, "nonpadding_tokens": 5859, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32419663899997886, "nonpadding_tokens": 8563, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3276664360000723, "nonpadding_tokens": 13490, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32663122399992517, "nonpadding_tokens": 1715, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3278734790000044, "nonpadding_tokens": 9738, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3268869019999556, "nonpadding_tokens": 5513, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3249216179999621, "nonpadding_tokens": 9919, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003841989999955331, "prefetch": {"data_wait_seconds": 0.0038434259994346576, "consumed_batches": 386, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 391, "tokens": 35592334, "loss_tokens": 46661, "loss": 4.319078700627934, "lr": 0.00029999346924202585, "optimizer_lrs": [0.019999564616135057, 0.00029999346924202585], "gradient_norm": 1.1266909837722778, "seconds": 0.2809618320000027, "nonpadding_tokens_per_second": 166075.93874174182, "ranks": [{"compute_seconds": 0.2759732230000509, "nonpadding_tokens": 4559, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.27866624099999626, "nonpadding_tokens": 6517, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2780369199999768, "nonpadding_tokens": 3091, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.27519097300000794, "nonpadding_tokens": 5749, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.27706727600002523, "nonpadding_tokens": 5406, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2753756950000934, "nonpadding_tokens": 9010, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2778220100000226, "nonpadding_tokens": 1894, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2764273800000865, "nonpadding_tokens": 10435, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003597200000058365, "prefetch": {"data_wait_seconds": 0.003852912999491309, "consumed_batches": 387, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 392, "tokens": 35680504, "loss_tokens": 88170, "loss": 4.184209127254168, "lr": 0.0002999933951748261, "optimizer_lrs": [0.019999559678321743, 0.0002999933951748261], "gradient_norm": 0.491916686296463, "seconds": 0.4096158379999224, "nonpadding_tokens_per_second": 215250.4659744546, "ranks": [{"compute_seconds": 0.3532505460000266, "nonpadding_tokens": 21079, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3570634789999758, "nonpadding_tokens": 18109, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3573309099999733, "nonpadding_tokens": 11508, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35724567600004775, "nonpadding_tokens": 9257, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3582309199999827, "nonpadding_tokens": 13705, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35784055700003137, "nonpadding_tokens": 6953, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.35976086900006976, "nonpadding_tokens": 4458, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36037680999993427, "nonpadding_tokens": 3101, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0276612310000246, "prefetch": {"data_wait_seconds": 0.003861381999399782, "consumed_batches": 388, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 393, "tokens": 35776612, "loss_tokens": 96108, "loss": 4.33141459347817, "lr": 0.00029999331396358454, "optimizer_lrs": [0.01999955426423897, 0.00029999331396358454], "gradient_norm": 0.4933604896068573, "seconds": 0.3914634449999994, "nonpadding_tokens_per_second": 245509.51366608482, "ranks": [{"compute_seconds": 0.3866696189999175, "nonpadding_tokens": 4814, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3891020309999931, "nonpadding_tokens": 11369, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3883265700000038, "nonpadding_tokens": 10108, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38622040600000673, "nonpadding_tokens": 20351, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.38782429899993076, "nonpadding_tokens": 9054, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.38614614300001904, "nonpadding_tokens": 6655, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.38650429300003, "nonpadding_tokens": 16661, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.38855224099995667, "nonpadding_tokens": 17096, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040296000008765986, "prefetch": {"data_wait_seconds": 0.0038716639994618163, "consumed_batches": 389, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 394, "tokens": 35866554, "loss_tokens": 89942, "loss": 4.188564227502168, "lr": 0.00029999323751312643, "optimizer_lrs": [0.019999549167541764, 0.00029999323751312643], "gradient_norm": 0.4581243395805359, "seconds": 0.31639821800001755, "nonpadding_tokens_per_second": 284268.3519791348, "ranks": [{"compute_seconds": 0.3119801349999989, "nonpadding_tokens": 5206, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31082580500003587, "nonpadding_tokens": 9301, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3138942920001, "nonpadding_tokens": 17762, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3114082240000471, "nonpadding_tokens": 17794, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31176229000004696, "nonpadding_tokens": 4259, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.311353218000022, "nonpadding_tokens": 15955, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.31190154600005826, "nonpadding_tokens": 11806, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.312717816999907, "nonpadding_tokens": 7859, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0005580079999845111, "prefetch": {"data_wait_seconds": 0.0038796919994865675, "consumed_batches": 390, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 395, "tokens": 35938924, "loss_tokens": 72370, "loss": 4.335159596517894, "lr": 0.00029999317568329715, "optimizer_lrs": [0.019999545045553145, 0.00029999317568329715], "gradient_norm": 0.5029858350753784, "seconds": 0.3684483659999387, "nonpadding_tokens_per_second": 196418.29542002105, "ranks": [{"compute_seconds": 0.3650671409999404, "nonpadding_tokens": 17750, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.36096829699999944, "nonpadding_tokens": 12919, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36246439799992913, "nonpadding_tokens": 4514, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36606292700002996, "nonpadding_tokens": 4163, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36226196400002664, "nonpadding_tokens": 13712, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36177403499993943, "nonpadding_tokens": 7015, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.36527335499999936, "nonpadding_tokens": 8025, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3614132369999652, "nonpadding_tokens": 4272, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036988499994095037, "prefetch": {"data_wait_seconds": 0.0038890429995035447, "consumed_batches": 391, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 396, "tokens": 36039879, "loss_tokens": 100955, "loss": 4.535422960725075, "lr": 0.00029999308896165263, "optimizer_lrs": [0.01999953926411018, 0.00029999308896165263], "gradient_norm": 0.5852177143096924, "seconds": 0.3046130999999832, "nonpadding_tokens_per_second": 331420.41494605964, "ranks": [{"compute_seconds": 0.30126988600000004, "nonpadding_tokens": 15527, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.302200765000066, "nonpadding_tokens": 7983, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30100021300006574, "nonpadding_tokens": 7440, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.29761820000010175, "nonpadding_tokens": 14286, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30160961700005373, "nonpadding_tokens": 10276, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.29845315800002936, "nonpadding_tokens": 14560, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2990233460000127, "nonpadding_tokens": 18776, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.29872338600000603, "nonpadding_tokens": 12107, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004168399999571193, "prefetch": {"data_wait_seconds": 0.0038993949995074217, "consumed_batches": 392, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 397, "tokens": 36154410, "loss_tokens": 114531, "loss": 4.526035254647214, "lr": 0.0002999929899151239, "optimizer_lrs": [0.019999532661008262, 0.0002999929899151239], "gradient_norm": 0.5859041213989258, "seconds": 0.35621702400010236, "nonpadding_tokens_per_second": 321520.2875872858, "ranks": [{"compute_seconds": 0.3524247400000604, "nonpadding_tokens": 9359, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34932593700000325, "nonpadding_tokens": 12037, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35010514300006434, "nonpadding_tokens": 28023, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3538148720000436, "nonpadding_tokens": 8800, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3520085320000135, "nonpadding_tokens": 11986, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35383742399994844, "nonpadding_tokens": 26133, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.35074711300001127, "nonpadding_tokens": 11599, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3525003340000694, "nonpadding_tokens": 6594, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003581000000849599, "prefetch": {"data_wait_seconds": 0.003910301999553667, "consumed_batches": 393, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 398, "tokens": 36251948, "loss_tokens": 97538, "loss": 3.79253438403494, "lr": 0.00029999290500854517, "optimizer_lrs": [0.01999952700056968, 0.00029999290500854517], "gradient_norm": 0.5114414691925049, "seconds": 0.4410879020000493, "nonpadding_tokens_per_second": 221130.52649534942, "ranks": [{"compute_seconds": 0.4384296279999944, "nonpadding_tokens": 16229, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4382243990000916, "nonpadding_tokens": 25406, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.437331208000046, "nonpadding_tokens": 6697, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4384048699999994, "nonpadding_tokens": 6046, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4352367999999842, "nonpadding_tokens": 4713, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.434597106999945, "nonpadding_tokens": 9393, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4375558420000516, "nonpadding_tokens": 13356, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.43518839600005776, "nonpadding_tokens": 15698, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037936199998966913, "prefetch": {"data_wait_seconds": 0.003920632999552254, "consumed_batches": 394, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 399, "tokens": 36328687, "loss_tokens": 76739, "loss": 4.067685596632742, "lr": 0.00029999283784820787, "optimizer_lrs": [0.01999952252321386, 0.00029999283784820787], "gradient_norm": 0.6373230218887329, "seconds": 0.2948381749999953, "nonpadding_tokens_per_second": 260274.97965621724, "ranks": [{"compute_seconds": 0.2910771659999227, "nonpadding_tokens": 6929, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2925057370000559, "nonpadding_tokens": 13630, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.291357539000046, "nonpadding_tokens": 19588, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.28805694699997275, "nonpadding_tokens": 8102, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.29146589600009065, "nonpadding_tokens": 10905, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2890637790000028, "nonpadding_tokens": 5824, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2892229149999821, "nonpadding_tokens": 2796, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.29176124200000686, "nonpadding_tokens": 8965, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038274400003501796, "prefetch": {"data_wait_seconds": 0.0039322149995086875, "consumed_batches": 395, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 400, "tokens": 36405972, "loss_tokens": 77285, "loss": 4.234444749951479, "lr": 0.00029999276989027687, "optimizer_lrs": [0.019999517992685126, 0.00029999276989027687], "gradient_norm": 0.4162936210632324, "seconds": 0.3779392530000223, "nonpadding_tokens_per_second": 204490.5348849685, "ranks": [{"compute_seconds": 0.3737742399999888, "nonpadding_tokens": 11017, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3753548340000634, "nonpadding_tokens": 8730, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.37170120600001155, "nonpadding_tokens": 13124, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3755548430000317, "nonpadding_tokens": 8412, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3731715090000307, "nonpadding_tokens": 9854, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3710752649999449, "nonpadding_tokens": 12316, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.37265806799996426, "nonpadding_tokens": 1130, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.372366869000075, "nonpadding_tokens": 12702, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003697310000916332, "prefetch": {"data_wait_seconds": 0.003942880999488807, "consumed_batches": 396, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 401, "tokens": 36495789, "loss_tokens": 89817, "loss": 4.399022108843537, "lr": 0.00029999269050961326, "optimizer_lrs": [0.019999512700640886, 0.00029999269050961326], "gradient_norm": 0.416509747505188, "seconds": 0.3553779780000923, "nonpadding_tokens_per_second": 252736.5384468946, "ranks": [{"compute_seconds": 0.35240023700009715, "nonpadding_tokens": 17691, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3528274730000476, "nonpadding_tokens": 13362, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34930123499998444, "nonpadding_tokens": 11084, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3530359669999825, "nonpadding_tokens": 7187, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3493656869999313, "nonpadding_tokens": 11216, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34906220700008816, "nonpadding_tokens": 10956, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3520940390000078, "nonpadding_tokens": 14552, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34936349900010555, "nonpadding_tokens": 3769, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039010000000416767, "prefetch": {"data_wait_seconds": 0.003952849999450336, "consumed_batches": 397, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 402, "tokens": 36580310, "loss_tokens": 84521, "loss": 4.135854920670603, "lr": 0.0002999926154137759, "optimizer_lrs": [0.01999950769425173, 0.0002999926154137759], "gradient_norm": 0.6106767654418945, "seconds": 0.2917627059999859, "nonpadding_tokens_per_second": 289690.8969579001, "ranks": [{"compute_seconds": 0.287382158000014, "nonpadding_tokens": 2226, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.28937037100001817, "nonpadding_tokens": 18556, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2885760899999923, "nonpadding_tokens": 2049, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.28944112599992877, "nonpadding_tokens": 5384, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2857072119999202, "nonpadding_tokens": 6039, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2851221359999272, "nonpadding_tokens": 17435, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2884717570000248, "nonpadding_tokens": 14814, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.28569438900001387, "nonpadding_tokens": 18018, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003681399999777568, "prefetch": {"data_wait_seconds": 0.003962571999409192, "consumed_batches": 398, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 403, "tokens": 36690104, "loss_tokens": 109794, "loss": 3.8178241752736946, "lr": 0.0002999925172901306, "optimizer_lrs": [0.019999501152675375, 0.0002999925172901306], "gradient_norm": 0.552185595035553, "seconds": 0.3062477230000695, "nonpadding_tokens_per_second": 358513.68599392025, "ranks": [{"compute_seconds": 0.30378585400001157, "nonpadding_tokens": 11327, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3024208289999706, "nonpadding_tokens": 11886, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30299728999989384, "nonpadding_tokens": 19802, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30139455899995937, "nonpadding_tokens": 18207, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30075608000004195, "nonpadding_tokens": 11159, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.29754989499997464, "nonpadding_tokens": 13540, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.30133421099992574, "nonpadding_tokens": 12052, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30097486600004686, "nonpadding_tokens": 11821, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003811179999502201, "prefetch": {"data_wait_seconds": 0.003972148999423553, "consumed_batches": 399, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 404, "tokens": 36772594, "loss_tokens": 82490, "loss": 4.0105846163171295, "lr": 0.00029999244314220144, "optimizer_lrs": [0.0199994962094801, 0.00029999244314220144], "gradient_norm": 0.577010989189148, "seconds": 0.3141738740000619, "nonpadding_tokens_per_second": 262561.61580126727, "ranks": [{"compute_seconds": 0.3116881830000011, "nonpadding_tokens": 11657, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30743046399993545, "nonpadding_tokens": 6322, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3087545930000033, "nonpadding_tokens": 13333, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30764846299996407, "nonpadding_tokens": 3195, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3101175529999409, "nonpadding_tokens": 6391, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30760600199994315, "nonpadding_tokens": 4190, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3092869570000403, "nonpadding_tokens": 23683, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3105167410000149, "nonpadding_tokens": 13719, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040890300010687497, "prefetch": {"data_wait_seconds": 0.003983483999377313, "consumed_batches": 400, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 405, "tokens": 36848081, "loss_tokens": 75487, "loss": 4.232072244227483, "lr": 0.00029999237496874915, "optimizer_lrs": [0.01999949166458328, 0.00029999237496874915], "gradient_norm": 0.602769136428833, "seconds": 0.38746812800002317, "nonpadding_tokens_per_second": 194821.18539565528, "ranks": [{"compute_seconds": 0.38230998500000624, "nonpadding_tokens": 9415, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.38042377799990845, "nonpadding_tokens": 3578, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.38021576899996035, "nonpadding_tokens": 2570, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38278780400003143, "nonpadding_tokens": 23910, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.38157072800004244, "nonpadding_tokens": 8672, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.38501433800001905, "nonpadding_tokens": 941, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.38197480599990286, "nonpadding_tokens": 13705, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3812862199999927, "nonpadding_tokens": 12696, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003965529999732098, "prefetch": {"data_wait_seconds": 0.003995080999402489, "consumed_batches": 401, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 406, "tokens": 36929551, "loss_tokens": 81470, "loss": 4.340094513317785, "lr": 0.0002999923010484812, "optimizer_lrs": [0.019999486736565413, 0.0002999923010484812], "gradient_norm": 0.48781484365463257, "seconds": 0.47690933000001223, "nonpadding_tokens_per_second": 170829.11755993095, "ranks": [{"compute_seconds": 0.47135939800000415, "nonpadding_tokens": 18445, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.467094508999935, "nonpadding_tokens": 9396, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.47386966999999913, "nonpadding_tokens": 21949, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.46784189300001344, "nonpadding_tokens": 1780, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.46622190300001876, "nonpadding_tokens": 13484, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4638475220000373, "nonpadding_tokens": 1322, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4646707320000587, "nonpadding_tokens": 10247, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4645564420000028, "nonpadding_tokens": 4847, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00042251599995779543, "prefetch": {"data_wait_seconds": 0.004005152999297934, "consumed_batches": 402, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 407, "tokens": 37021508, "loss_tokens": 91957, "loss": 4.136761271572583, "lr": 0.0002999922171846591, "optimizer_lrs": [0.01999948114564394, 0.0002999922171846591], "gradient_norm": 0.758236825466156, "seconds": 0.3447211380000681, "nonpadding_tokens_per_second": 266757.6480325434, "ranks": [{"compute_seconds": 0.31964590799998405, "nonpadding_tokens": 4237, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3211334129999841, "nonpadding_tokens": 11347, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3211620980000589, "nonpadding_tokens": 18189, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31761231999996653, "nonpadding_tokens": 17933, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32629234800003815, "nonpadding_tokens": 12143, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31804946000011114, "nonpadding_tokens": 14659, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3193466410000383, "nonpadding_tokens": 4706, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32020240500003183, "nonpadding_tokens": 8743, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000656576999972458, "prefetch": {"data_wait_seconds": 0.004017527999280901, "consumed_batches": 403, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 408, "tokens": 37071084, "loss_tokens": 49576, "loss": 4.257837398640471, "lr": 0.00029999217178339275, "optimizer_lrs": [0.01999947811889285, 0.00029999217178339275], "gradient_norm": 0.5228954553604126, "seconds": 0.4086704069999314, "nonpadding_tokens_per_second": 121310.47208419062, "ranks": [{"compute_seconds": 0.4029270280000219, "nonpadding_tokens": 2658, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40575577999993584, "nonpadding_tokens": 10831, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.40418269999997847, "nonpadding_tokens": 6599, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.400076747000071, "nonpadding_tokens": 6528, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4017191330000287, "nonpadding_tokens": 8539, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.40617644899998595, "nonpadding_tokens": 5286, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.40464302500004123, "nonpadding_tokens": 6482, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4045817859999943, "nonpadding_tokens": 2653, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003992279999920356, "prefetch": {"data_wait_seconds": 0.004026144999329517, "consumed_batches": 404, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 409, "tokens": 37194292, "loss_tokens": 123208, "loss": 4.019675061684306, "lr": 0.0002999920583787625, "optimizer_lrs": [0.019999470558584167, 0.0002999920583787625], "gradient_norm": 0.5353615880012512, "seconds": 0.5038921379999692, "nonpadding_tokens_per_second": 244512.6460774578, "ranks": [{"compute_seconds": 0.4996055440000191, "nonpadding_tokens": 19748, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.49549010600003385, "nonpadding_tokens": 13264, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.49908340199999657, "nonpadding_tokens": 21833, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4999920340000017, "nonpadding_tokens": 18549, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.5015739619999522, "nonpadding_tokens": 18644, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.501032953000049, "nonpadding_tokens": 6610, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4973422790000086, "nonpadding_tokens": 3929, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4987456700000621, "nonpadding_tokens": 20631, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034955099999933736, "prefetch": {"data_wait_seconds": 0.004035798999439066, "consumed_batches": 405, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 410, "tokens": 37317040, "loss_tokens": 122748, "loss": 4.114044628018379, "lr": 0.0002999919445865866, "optimizer_lrs": [0.01999946297243911, 0.0002999919445865866], "gradient_norm": 0.48029977083206177, "seconds": 0.32762092100006157, "nonpadding_tokens_per_second": 374664.71806901775, "ranks": [{"compute_seconds": 0.3247287699999788, "nonpadding_tokens": 10748, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3211457419999988, "nonpadding_tokens": 24137, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3247239869999703, "nonpadding_tokens": 7081, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32527321500003836, "nonpadding_tokens": 15826, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32197723199999473, "nonpadding_tokens": 20680, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32219038799996724, "nonpadding_tokens": 11116, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.32498781900005724, "nonpadding_tokens": 20539, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3217034799999965, "nonpadding_tokens": 12621, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004009040000028108, "prefetch": {"data_wait_seconds": 0.004045769999379445, "consumed_batches": 406, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 411, "tokens": 37419663, "loss_tokens": 102623, "loss": 4.28718525574189, "lr": 0.0002999918488298262, "optimizer_lrs": [0.019999456588655082, 0.0002999918488298262], "gradient_norm": 0.5200923681259155, "seconds": 0.3077640080000492, "nonpadding_tokens_per_second": 333447.04816810024, "ranks": [{"compute_seconds": 0.3052610760000789, "nonpadding_tokens": 9420, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30072311300000365, "nonpadding_tokens": 7969, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3020831299999145, "nonpadding_tokens": 17755, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30154024399996615, "nonpadding_tokens": 12391, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3023600989999977, "nonpadding_tokens": 26829, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30147908600008577, "nonpadding_tokens": 16123, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.30298077300005843, "nonpadding_tokens": 7012, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3049805020000349, "nonpadding_tokens": 5124, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040265299992370274, "prefetch": {"data_wait_seconds": 0.004055736999362125, "consumed_batches": 407, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 412, "tokens": 37548274, "loss_tokens": 128611, "loss": 4.000083099423844, "lr": 0.0002999917280250366, "optimizer_lrs": [0.019999448535002445, 0.0002999917280250366], "gradient_norm": 0.5167157053947449, "seconds": 0.46801226500008397, "nonpadding_tokens_per_second": 274802.62723451684, "ranks": [{"compute_seconds": 0.4150346850000233, "nonpadding_tokens": 1704, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.42886148900004173, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.43278703400005725, "nonpadding_tokens": 11788, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.43401470200001313, "nonpadding_tokens": 22262, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.43224447099998997, "nonpadding_tokens": 12845, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4304822999999942, "nonpadding_tokens": 9370, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4309424979999221, "nonpadding_tokens": 14778, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4327909549999731, "nonpadding_tokens": 23096, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.031657626999958666, "prefetch": {"data_wait_seconds": 0.0040645199993605274, "consumed_batches": 408, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 413, "tokens": 37657098, "loss_tokens": 108824, "loss": 4.195365337609351, "lr": 0.00029999162511220456, "optimizer_lrs": [0.01999944167414697, 0.00029999162511220456], "gradient_norm": 0.6535966992378235, "seconds": 0.30825227599996197, "nonpadding_tokens_per_second": 353035.51173135027, "ranks": [{"compute_seconds": 0.3044696979999344, "nonpadding_tokens": 15430, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3058426690000715, "nonpadding_tokens": 6489, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30203090600002724, "nonpadding_tokens": 6538, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30538603200000125, "nonpadding_tokens": 11165, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3039868660000593, "nonpadding_tokens": 22334, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30108401600000434, "nonpadding_tokens": 18004, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3042244829999845, "nonpadding_tokens": 17303, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30144665299997087, "nonpadding_tokens": 11561, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039962400001058995, "prefetch": {"data_wait_seconds": 0.00407399999937752, "consumed_batches": 409, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 414, "tokens": 37774930, "loss_tokens": 117832, "loss": 4.341640534829248, "lr": 0.00029999151296329597, "optimizer_lrs": [0.019999434197553064, 0.00029999151296329597], "gradient_norm": 0.47848379611968994, "seconds": 0.41727218099993024, "nonpadding_tokens_per_second": 282386.4263312097, "ranks": [{"compute_seconds": 0.413183573000083, "nonpadding_tokens": 26699, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4089095140000154, "nonpadding_tokens": 14003, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.40990498799999386, "nonpadding_tokens": 20783, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.40987056100004793, "nonpadding_tokens": 9199, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.41184599300004265, "nonpadding_tokens": 7446, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4148664330000429, "nonpadding_tokens": 10123, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4108068129999083, "nonpadding_tokens": 4033, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.41075965799996084, "nonpadding_tokens": 25546, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003736940000180766, "prefetch": {"data_wait_seconds": 0.004082908999293977, "consumed_batches": 410, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 415, "tokens": 37861800, "loss_tokens": 86870, "loss": 4.280438442500288, "lr": 0.00029999142980542627, "optimizer_lrs": [0.019999428653695087, 0.00029999142980542627], "gradient_norm": 0.5906129479408264, "seconds": 0.32129574399994, "nonpadding_tokens_per_second": 270373.95179444464, "ranks": [{"compute_seconds": 0.3105261260000134, "nonpadding_tokens": 3808, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31482070799995654, "nonpadding_tokens": 4109, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3180875799999967, "nonpadding_tokens": 8284, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3155400269999973, "nonpadding_tokens": 8055, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31659243799992964, "nonpadding_tokens": 13593, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3159863390000055, "nonpadding_tokens": 18371, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3188472399999682, "nonpadding_tokens": 17909, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31895183200003885, "nonpadding_tokens": 12741, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040370099998199294, "prefetch": {"data_wait_seconds": 0.004095201999348319, "consumed_batches": 411, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 416, "tokens": 37935343, "loss_tokens": 73543, "loss": 3.6166749384713706, "lr": 0.00029999135908818453, "optimizer_lrs": [0.019999423939212304, 0.00029999135908818453], "gradient_norm": 0.4706200659275055, "seconds": 0.30860383599997476, "nonpadding_tokens_per_second": 238308.7681385983, "ranks": [{"compute_seconds": 0.3056706849999955, "nonpadding_tokens": 8202, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3000984760000165, "nonpadding_tokens": 12816, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.299711904999981, "nonpadding_tokens": 11006, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30363250900006733, "nonpadding_tokens": 6057, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3019354519999524, "nonpadding_tokens": 6769, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.306182874000001, "nonpadding_tokens": 9852, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.30230653799992524, "nonpadding_tokens": 7600, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30197533400007615, "nonpadding_tokens": 11241, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003868459999694096, "prefetch": {"data_wait_seconds": 0.004105542999354839, "consumed_batches": 412, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 417, "tokens": 38053356, "loss_tokens": 118013, "loss": 4.2537818926728415, "lr": 0.0002999912450024079, "optimizer_lrs": [0.01999941633349386, 0.0002999912450024079], "gradient_norm": 0.5206950902938843, "seconds": 0.37165693599990846, "nonpadding_tokens_per_second": 317532.0801762975, "ranks": [{"compute_seconds": 0.3669203860000607, "nonpadding_tokens": 18787, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.36592732000008255, "nonpadding_tokens": 2004, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3665351990000545, "nonpadding_tokens": 10015, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36605859700000565, "nonpadding_tokens": 13859, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36717271800000617, "nonpadding_tokens": 24277, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36643167300007917, "nonpadding_tokens": 20119, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3685701560000325, "nonpadding_tokens": 13631, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36931945600008476, "nonpadding_tokens": 15321, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035939500003223657, "prefetch": {"data_wait_seconds": 0.004114000999379641, "consumed_batches": 413, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 418, "tokens": 38138385, "loss_tokens": 85029, "loss": 3.684735134483529, "lr": 0.0002999911623392587, "optimizer_lrs": [0.019999410822617247, 0.0002999911623392587], "gradient_norm": 0.4736720025539398, "seconds": 0.35678651800003536, "nonpadding_tokens_per_second": 238318.98266960742, "ranks": [{"compute_seconds": 0.35152964700000666, "nonpadding_tokens": 13677, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3541223040000432, "nonpadding_tokens": 12580, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3500383220000458, "nonpadding_tokens": 4160, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3542076869999846, "nonpadding_tokens": 7945, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34993502899999385, "nonpadding_tokens": 11459, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34998794000000544, "nonpadding_tokens": 9196, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.350907535000033, "nonpadding_tokens": 20454, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3529895010000246, "nonpadding_tokens": 5558, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034947899996495835, "prefetch": {"data_wait_seconds": 0.004123643999378146, "consumed_batches": 414, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 419, "tokens": 38238018, "loss_tokens": 99633, "loss": 4.004160268184236, "lr": 0.00029999106498425815, "optimizer_lrs": [0.01999940433228388, 0.00029999106498425815], "gradient_norm": 0.4961880147457123, "seconds": 0.3549784479999971, "nonpadding_tokens_per_second": 280673.37766939815, "ranks": [{"compute_seconds": 0.3461752419999584, "nonpadding_tokens": 3652, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3472265810000863, "nonpadding_tokens": 18308, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3460393960000374, "nonpadding_tokens": 14996, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34739319300001625, "nonpadding_tokens": 15015, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3463566750000382, "nonpadding_tokens": 7045, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.343551651000098, "nonpadding_tokens": 19226, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3526207790000626, "nonpadding_tokens": 8080, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3441756690000375, "nonpadding_tokens": 13311, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003856440000618022, "prefetch": {"data_wait_seconds": 0.004131902999347403, "consumed_batches": 415, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 420, "tokens": 38315812, "loss_tokens": 77794, "loss": 4.11505707381032, "lr": 0.00029999098859818214, "optimizer_lrs": [0.01999939923987881, 0.00029999098859818214], "gradient_norm": 0.4775582551956177, "seconds": 0.336717206000003, "nonpadding_tokens_per_second": 231036.6046456186, "ranks": [{"compute_seconds": 0.33377680099999907, "nonpadding_tokens": 16826, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33435432500004936, "nonpadding_tokens": 4242, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33422415900008673, "nonpadding_tokens": 12184, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33444330400004674, "nonpadding_tokens": 13849, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.333936934999997, "nonpadding_tokens": 10144, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33104241600005935, "nonpadding_tokens": 7248, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3341838660001031, "nonpadding_tokens": 5496, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3316835309999533, "nonpadding_tokens": 7805, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003535670000474056, "prefetch": {"data_wait_seconds": 0.0041413429993326645, "consumed_batches": 416, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 421, "tokens": 38379584, "loss_tokens": 63772, "loss": 4.105272739211566, "lr": 0.00029999092573783836, "optimizer_lrs": [0.019999395049189225, 0.00029999092573783836], "gradient_norm": 0.5270116925239563, "seconds": 0.28567738599997483, "nonpadding_tokens_per_second": 223230.83003848832, "ranks": [{"compute_seconds": 0.2832774610000115, "nonpadding_tokens": 3676, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.27917923000006795, "nonpadding_tokens": 9194, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2796161790000724, "nonpadding_tokens": 9785, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2794608450000169, "nonpadding_tokens": 16754, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2819539939999913, "nonpadding_tokens": 3856, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.27906083000004855, "nonpadding_tokens": 7932, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2803522259999909, "nonpadding_tokens": 8000, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2793545519999725, "nonpadding_tokens": 4575, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004090290000249297, "prefetch": {"data_wait_seconds": 0.004151904999275757, "consumed_batches": 417, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 422, "tokens": 38461487, "loss_tokens": 81903, "loss": 3.865859232873033, "lr": 0.00029999084468520766, "optimizer_lrs": [0.019999389645680512, 0.00029999084468520766], "gradient_norm": 0.43318164348602295, "seconds": 0.32212979400003405, "nonpadding_tokens_per_second": 254254.65612159844, "ranks": [{"compute_seconds": 0.31845589399995333, "nonpadding_tokens": 4055, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3197476599999618, "nonpadding_tokens": 13743, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3181925310000224, "nonpadding_tokens": 18382, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3151456439999265, "nonpadding_tokens": 10854, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3187458149999429, "nonpadding_tokens": 11187, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31573496100008924, "nonpadding_tokens": 6856, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3188740370000005, "nonpadding_tokens": 4061, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31553300099994885, "nonpadding_tokens": 12765, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003957369999625371, "prefetch": {"data_wait_seconds": 0.0041633869992665495, "consumed_batches": 418, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 423, "tokens": 38551448, "loss_tokens": 89961, "loss": 4.060707903980614, "lr": 0.0002999907552429473, "optimizer_lrs": [0.019999383682863157, 0.0002999907552429473], "gradient_norm": 0.5345409512519836, "seconds": 0.3655590309999752, "nonpadding_tokens_per_second": 246091.5813074417, "ranks": [{"compute_seconds": 0.3613660290000098, "nonpadding_tokens": 11937, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3621356569999534, "nonpadding_tokens": 13826, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3586241579999978, "nonpadding_tokens": 1189, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3579232270000148, "nonpadding_tokens": 13475, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35837514099989676, "nonpadding_tokens": 11563, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3631188199999542, "nonpadding_tokens": 11799, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.35941709400003674, "nonpadding_tokens": 20714, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3608491380000487, "nonpadding_tokens": 5458, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003935189999992872, "prefetch": {"data_wait_seconds": 0.004174421999209699, "consumed_batches": 419, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 424, "tokens": 38632480, "loss_tokens": 81032, "loss": 3.919716901964656, "lr": 0.00029999067430602256, "optimizer_lrs": [0.019999378287068173, 0.00029999067430602256], "gradient_norm": 0.5241541862487793, "seconds": 0.37852019800004655, "nonpadding_tokens_per_second": 214075.76247751524, "ranks": [{"compute_seconds": 0.3747048809999569, "nonpadding_tokens": 23886, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3753456990000359, "nonpadding_tokens": 15521, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3722733710000057, "nonpadding_tokens": 14052, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37614715400002297, "nonpadding_tokens": 1589, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37508283399995435, "nonpadding_tokens": 5372, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3722376489999988, "nonpadding_tokens": 7915, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3727994219999573, "nonpadding_tokens": 8197, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3741685080000252, "nonpadding_tokens": 4500, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003579849999368889, "prefetch": {"data_wait_seconds": 0.004184525999221478, "consumed_batches": 420, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 425, "tokens": 38707266, "loss_tokens": 74786, "loss": 4.0077274657021364, "lr": 0.0002999905992947632, "optimizer_lrs": [0.019999373286317547, 0.0002999905992947632], "gradient_norm": 0.48826169967651367, "seconds": 0.34117267900001025, "nonpadding_tokens_per_second": 219202.78088855336, "ranks": [{"compute_seconds": 0.33868500699998094, "nonpadding_tokens": 3949, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33816900599993005, "nonpadding_tokens": 11857, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3348718510000026, "nonpadding_tokens": 6098, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33865629299998545, "nonpadding_tokens": 3244, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33726288599996224, "nonpadding_tokens": 5106, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3354395520000253, "nonpadding_tokens": 19404, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.33743604599999344, "nonpadding_tokens": 12831, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.335011312000006, "nonpadding_tokens": 12297, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000388466000003973, "prefetch": {"data_wait_seconds": 0.0041949749992227225, "consumed_batches": 421, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 426, "tokens": 38778470, "loss_tokens": 71204, "loss": 4.146419530503905, "lr": 0.0002999905275970759, "optimizer_lrs": [0.01999936850647173, 0.0002999905275970759], "gradient_norm": 0.6497959494590759, "seconds": 0.3537193509999952, "nonpadding_tokens_per_second": 201300.832987226, "ranks": [{"compute_seconds": 0.34960932200010575, "nonpadding_tokens": 10266, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35135631899993314, "nonpadding_tokens": 6700, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3504438569999593, "nonpadding_tokens": 14081, "peak_allocated_bytes": 14989288960, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34764534400005687, "nonpadding_tokens": 1442, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34821206899994195, "nonpadding_tokens": 10989, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34758413999998083, "nonpadding_tokens": 22873, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3507154190000392, "nonpadding_tokens": 1291, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3504054459999679, "nonpadding_tokens": 3562, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040219200002411526, "prefetch": {"data_wait_seconds": 0.00420641399921351, "consumed_batches": 422, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 427, "tokens": 38878254, "loss_tokens": 99784, "loss": 3.8347731099174216, "lr": 0.0002999904266629409, "optimizer_lrs": [0.019999361777529395, 0.0002999904266629409], "gradient_norm": 0.664895236492157, "seconds": 0.29939011399994797, "nonpadding_tokens_per_second": 333290.8981758073, "ranks": [{"compute_seconds": 0.2943587890000572, "nonpadding_tokens": 1569, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2961224409999659, "nonpadding_tokens": 4757, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2942962310000894, "nonpadding_tokens": 17939, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.29695877899996503, "nonpadding_tokens": 12899, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2945925660000057, "nonpadding_tokens": 25469, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.29160950300001787, "nonpadding_tokens": 24055, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.29348041999992347, "nonpadding_tokens": 7790, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2946059079999941, "nonpadding_tokens": 5306, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034614499998042447, "prefetch": {"data_wait_seconds": 0.004215872999225212, "consumed_batches": 423, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 428, "tokens": 38965460, "loss_tokens": 87206, "loss": 4.044332887071991, "lr": 0.00029999033801377835, "optimizer_lrs": [0.019999355867585226, 0.00029999033801377835], "gradient_norm": 0.5521703362464905, "seconds": 0.36645212500002344, "nonpadding_tokens_per_second": 237973.78716249613, "ranks": [{"compute_seconds": 0.36014232800005175, "nonpadding_tokens": 15002, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.36297090699997625, "nonpadding_tokens": 4061, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3619077150000294, "nonpadding_tokens": 8905, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3630627680000771, "nonpadding_tokens": 11580, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35881366700004946, "nonpadding_tokens": 16910, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3639340519999905, "nonpadding_tokens": 5545, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.36231763000000683, "nonpadding_tokens": 18335, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3615592899999456, "nonpadding_tokens": 6868, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004912259998945956, "prefetch": {"data_wait_seconds": 0.004225796999321574, "consumed_batches": 424, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 429, "tokens": 39078575, "loss_tokens": 113115, "loss": 3.993363778013526, "lr": 0.00029999022241821067, "optimizer_lrs": [0.019999348161214046, 0.00029999022241821067], "gradient_norm": 0.45033901929855347, "seconds": 0.3220413000000235, "nonpadding_tokens_per_second": 351243.76904450374, "ranks": [{"compute_seconds": 0.319136694000008, "nonpadding_tokens": 6323, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31968577500003903, "nonpadding_tokens": 20580, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31812335600000097, "nonpadding_tokens": 22449, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31960462100005316, "nonpadding_tokens": 3840, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31606323200003317, "nonpadding_tokens": 10428, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31532912900001975, "nonpadding_tokens": 25608, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.31636793699999544, "nonpadding_tokens": 6751, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31852084000001923, "nonpadding_tokens": 17136, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00041900600001554267, "prefetch": {"data_wait_seconds": 0.004235938999386235, "consumed_batches": 425, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 430, "tokens": 39150258, "loss_tokens": 71683, "loss": 4.026146802589177, "lr": 0.0002999901488074134, "optimizer_lrs": [0.019999343253827562, 0.0002999901488074134], "gradient_norm": 0.668298602104187, "seconds": 0.35288705200002823, "nonpadding_tokens_per_second": 203132.98431815024, "ranks": [{"compute_seconds": 0.3490470290000758, "nonpadding_tokens": 1989, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3503473010000562, "nonpadding_tokens": 7615, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34870291499998984, "nonpadding_tokens": 10903, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.345997728000043, "nonpadding_tokens": 3776, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34923219500001323, "nonpadding_tokens": 19596, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35056205899991255, "nonpadding_tokens": 10151, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3492634880000196, "nonpadding_tokens": 2088, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34935015500002464, "nonpadding_tokens": 15565, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003877870000223993, "prefetch": {"data_wait_seconds": 0.004245801999445575, "consumed_batches": 426, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 431, "tokens": 39234289, "loss_tokens": 84031, "loss": 3.6793523372326877, "lr": 0.00029999006216507325, "optimizer_lrs": [0.01999933747767155, 0.00029999006216507325], "gradient_norm": 0.5500871539115906, "seconds": 0.2659930869999698, "nonpadding_tokens_per_second": 315914.2252445438, "ranks": [{"compute_seconds": 0.2628472100000181, "nonpadding_tokens": 3456, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.26052239799992094, "nonpadding_tokens": 2949, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.26097352800002227, "nonpadding_tokens": 19049, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2604946190000419, "nonpadding_tokens": 9992, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2636155540000118, "nonpadding_tokens": 9660, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.26026133800007756, "nonpadding_tokens": 11571, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.263654603999953, "nonpadding_tokens": 17454, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2633839979999948, "nonpadding_tokens": 9900, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003875859999880049, "prefetch": {"data_wait_seconds": 0.004256820999444244, "consumed_batches": 427, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 432, "tokens": 39319420, "loss_tokens": 85131, "loss": 3.5811172633940633, "lr": 0.0002999899740017345, "optimizer_lrs": [0.019999331600115633, 0.0002999899740017345], "gradient_norm": 0.4909316599369049, "seconds": 0.4418089280000004, "nonpadding_tokens_per_second": 192687.36914252653, "ranks": [{"compute_seconds": 0.33570617400005176, "nonpadding_tokens": 14265, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3480061980000073, "nonpadding_tokens": 9902, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34889853600009246, "nonpadding_tokens": 9764, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3530594609999298, "nonpadding_tokens": 10677, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35134993199994824, "nonpadding_tokens": 5724, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34876152099991486, "nonpadding_tokens": 12868, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.34931488199993055, "nonpadding_tokens": 7003, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35193545200002063, "nonpadding_tokens": 14928, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.04051507799999854, "prefetch": {"data_wait_seconds": 0.004265842999416236, "consumed_batches": 428, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 433, "tokens": 39390487, "loss_tokens": 71067, "loss": 4.061262611338596, "lr": 0.0002999899001051863, "optimizer_lrs": [0.01999932667367909, 0.0002999899001051863], "gradient_norm": 0.5823153853416443, "seconds": 0.2443156319999389, "nonpadding_tokens_per_second": 290881.9195000088, "ranks": [{"compute_seconds": 0.238425588000041, "nonpadding_tokens": 9102, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.23669198099992173, "nonpadding_tokens": 3529, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.23968332599997666, "nonpadding_tokens": 4354, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.23685290399998848, "nonpadding_tokens": 15318, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.23962786299989602, "nonpadding_tokens": 7181, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.24180718499997056, "nonpadding_tokens": 7640, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.24015248599994266, "nonpadding_tokens": 14387, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2396596580000505, "nonpadding_tokens": 9556, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004199930000368113, "prefetch": {"data_wait_seconds": 0.0042759079993857085, "consumed_batches": 429, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 434, "tokens": 39467760, "loss_tokens": 77273, "loss": 3.8473581490300623, "lr": 0.0002999898194476529, "optimizer_lrs": [0.019999321296510195, 0.0002999898194476529], "gradient_norm": 0.5279854536056519, "seconds": 0.4318681560000641, "nonpadding_tokens_per_second": 178927.29280088094, "ranks": [{"compute_seconds": 0.3820707449999645, "nonpadding_tokens": 4921, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4102755880001041, "nonpadding_tokens": 13596, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4065221340000562, "nonpadding_tokens": 6442, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4105934359999992, "nonpadding_tokens": 8195, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.40921529599995665, "nonpadding_tokens": 10894, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.40639794200001234, "nonpadding_tokens": 17832, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.40727059499999996, "nonpadding_tokens": 10227, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.40908176799996454, "nonpadding_tokens": 5166, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0022569839999277974, "prefetch": {"data_wait_seconds": 0.0042851969993762395, "consumed_batches": 430, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 435, "tokens": 39538446, "loss_tokens": 70686, "loss": 3.9755688891718304, "lr": 0.0002999897453847053, "optimizer_lrs": [0.019999316358980355, 0.0002999897453847053], "gradient_norm": 0.46881526708602905, "seconds": 0.2961650060000238, "nonpadding_tokens_per_second": 238671.00625654036, "ranks": [{"compute_seconds": 0.29333808800004135, "nonpadding_tokens": 7840, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.29042437900000095, "nonpadding_tokens": 3435, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.29162905700002284, "nonpadding_tokens": 5385, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2899436190000415, "nonpadding_tokens": 6365, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.29317254299996875, "nonpadding_tokens": 14903, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2910684839999931, "nonpadding_tokens": 15627, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.29357621500003006, "nonpadding_tokens": 14361, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2938145099999474, "nonpadding_tokens": 2770, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040356699992116774, "prefetch": {"data_wait_seconds": 0.004295100999456736, "consumed_batches": 431, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 436, "tokens": 39654072, "loss_tokens": 115626, "loss": 4.185336505197793, "lr": 0.00029998962365615804, "optimizer_lrs": [0.01999930824374387, 0.00029998962365615804], "gradient_norm": 0.9219508171081543, "seconds": 0.3984058629999936, "nonpadding_tokens_per_second": 290221.6326068521, "ranks": [{"compute_seconds": 0.39571108699999513, "nonpadding_tokens": 22671, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3957174610000038, "nonpadding_tokens": 6705, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3929398199999241, "nonpadding_tokens": 11594, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.39148705399998107, "nonpadding_tokens": 8590, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3955396329999985, "nonpadding_tokens": 20484, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3960341779999226, "nonpadding_tokens": 16532, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3923792959999446, "nonpadding_tokens": 10220, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.39523101899999347, "nonpadding_tokens": 18830, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034444599998550984, "prefetch": {"data_wait_seconds": 0.004304501999513377, "consumed_batches": 432, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 437, "tokens": 39742522, "loss_tokens": 88450, "loss": 4.390703787450537, "lr": 0.00029998953005307155, "optimizer_lrs": [0.019999302003538104, 0.00029998953005307155], "gradient_norm": 2.6302037239074707, "seconds": 0.3855392509999547, "nonpadding_tokens_per_second": 229418.92367791728, "ranks": [{"compute_seconds": 0.3796011949999638, "nonpadding_tokens": 19456, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.38300544000003356, "nonpadding_tokens": 9261, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.379408556000044, "nonpadding_tokens": 10932, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.383211713000037, "nonpadding_tokens": 10726, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3819014819999893, "nonpadding_tokens": 12908, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3825677029999497, "nonpadding_tokens": 3355, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.38202930399995694, "nonpadding_tokens": 10776, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37994417999993857, "nonpadding_tokens": 11036, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003635619999613482, "prefetch": {"data_wait_seconds": 0.00431501999946704, "consumed_batches": 433, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 438, "tokens": 39860010, "loss_tokens": 117488, "loss": 3.8035687687253166, "lr": 0.00029998940507034274, "optimizer_lrs": [0.019999293671356184, 0.00029998940507034274], "gradient_norm": 0.675939679145813, "seconds": 0.4382040100000495, "nonpadding_tokens_per_second": 268112.5624568947, "ranks": [{"compute_seconds": 0.43317815000000337, "nonpadding_tokens": 9531, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.435839011999974, "nonpadding_tokens": 24734, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4332538529999965, "nonpadding_tokens": 8554, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.434868994999988, "nonpadding_tokens": 16313, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.42997086900004433, "nonpadding_tokens": 9697, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.43153049199997895, "nonpadding_tokens": 19271, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.430746140999986, "nonpadding_tokens": 22959, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.43389334300002247, "nonpadding_tokens": 6429, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035052800001267315, "prefetch": {"data_wait_seconds": 0.004324988999542256, "consumed_batches": 434, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 439, "tokens": 39948990, "loss_tokens": 88980, "loss": 3.834403447403911, "lr": 0.00029998930992071574, "optimizer_lrs": [0.01999928732804772, 0.00029998930992071574], "gradient_norm": 1.0164648294448853, "seconds": 0.30893198100000063, "nonpadding_tokens_per_second": 288024.5668058556, "ranks": [{"compute_seconds": 0.3035516649999863, "nonpadding_tokens": 6204, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3028671760000634, "nonpadding_tokens": 16733, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30345522200002506, "nonpadding_tokens": 12902, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3066165479999654, "nonpadding_tokens": 6366, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30342664900001637, "nonpadding_tokens": 5840, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.302832780000017, "nonpadding_tokens": 19772, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.30390315399995416, "nonpadding_tokens": 10760, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3032873550000659, "nonpadding_tokens": 10403, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003818829999318041, "prefetch": {"data_wait_seconds": 0.004333383999551188, "consumed_batches": 435, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 440, "tokens": 40038704, "loss_tokens": 89714, "loss": 4.162662042713512, "lr": 0.00029998921355558876, "optimizer_lrs": [0.019999280903705917, 0.00029998921355558876], "gradient_norm": 0.6908544898033142, "seconds": 0.3409760739999683, "nonpadding_tokens_per_second": 263109.3699554073, "ranks": [{"compute_seconds": 0.30332964699994136, "nonpadding_tokens": 10803, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31366756299996723, "nonpadding_tokens": 10509, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3139574030000176, "nonpadding_tokens": 5665, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.313920567000082, "nonpadding_tokens": 7532, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31461224700001367, "nonpadding_tokens": 17685, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31871344000001045, "nonpadding_tokens": 18217, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.31733141699999123, "nonpadding_tokens": 13414, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31654446400000325, "nonpadding_tokens": 5889, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.010895444000084353, "prefetch": {"data_wait_seconds": 0.004342284999552248, "consumed_batches": 436, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 441, "tokens": 40159465, "loss_tokens": 120761, "loss": 3.9300050823527464, "lr": 0.0002999890831590186, "optimizer_lrs": [0.01999927221060124, 0.0002999890831590186], "gradient_norm": 0.6344324946403503, "seconds": 0.3601563609998948, "nonpadding_tokens_per_second": 335301.58863426343, "ranks": [{"compute_seconds": 0.35279593799998565, "nonpadding_tokens": 19228, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35250685500000145, "nonpadding_tokens": 7735, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35293505199990705, "nonpadding_tokens": 6167, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35250418600003286, "nonpadding_tokens": 17415, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.353665761000002, "nonpadding_tokens": 25209, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3577471040000546, "nonpadding_tokens": 17182, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.35559441600003083, "nonpadding_tokens": 9393, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3557593470000029, "nonpadding_tokens": 18432, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034620999997514446, "prefetch": {"data_wait_seconds": 0.004350937999561211, "consumed_batches": 437, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 442, "tokens": 40281763, "loss_tokens": 122298, "loss": 4.030729243323685, "lr": 0.00029998895030437333, "optimizer_lrs": [0.01999926335362489, 0.00029998895030437333], "gradient_norm": 0.40093255043029785, "seconds": 0.3608567610000364, "nonpadding_tokens_per_second": 338910.0973501995, "ranks": [{"compute_seconds": 0.3542464930000051, "nonpadding_tokens": 17025, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3530870679999225, "nonpadding_tokens": 15396, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3540708340000265, "nonpadding_tokens": 17869, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3584506079999983, "nonpadding_tokens": 12123, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.354791829000078, "nonpadding_tokens": 16942, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35379869500002314, "nonpadding_tokens": 19160, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3548392339999964, "nonpadding_tokens": 9923, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3565325600000051, "nonpadding_tokens": 13860, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003447939999432492, "prefetch": {"data_wait_seconds": 0.004360155999620474, "consumed_batches": 438, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 443, "tokens": 40383821, "loss_tokens": 102058, "loss": 4.123164034176645, "lr": 0.0002999888388217985, "optimizer_lrs": [0.019999255921453234, 0.0002999888388217985], "gradient_norm": 0.6501290798187256, "seconds": 0.32595564899997953, "nonpadding_tokens_per_second": 313103.94623658265, "ranks": [{"compute_seconds": 0.3193445319999455, "nonpadding_tokens": 9775, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3181017619999693, "nonpadding_tokens": 14244, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3214890450000212, "nonpadding_tokens": 9853, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3236236310000322, "nonpadding_tokens": 6840, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32163611500004663, "nonpadding_tokens": 14562, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32257180999999946, "nonpadding_tokens": 947, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.32248661599999195, "nonpadding_tokens": 19278, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3215387070000588, "nonpadding_tokens": 26559, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035729400008222, "prefetch": {"data_wait_seconds": 0.004369295999708811, "consumed_batches": 439, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 444, "tokens": 40472086, "loss_tokens": 88265, "loss": 3.784664079759814, "lr": 0.00029998874195472173, "optimizer_lrs": [0.019999249463648117, 0.00029998874195472173], "gradient_norm": 0.5579204559326172, "seconds": 0.33227496200004225, "nonpadding_tokens_per_second": 265638.432305317, "ranks": [{"compute_seconds": 0.327546773999984, "nonpadding_tokens": 5131, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32585990499990203, "nonpadding_tokens": 9607, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.327445584999964, "nonpadding_tokens": 13482, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3268187170000374, "nonpadding_tokens": 12135, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3295826119999674, "nonpadding_tokens": 10749, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3263049109999656, "nonpadding_tokens": 12911, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3299501770000006, "nonpadding_tokens": 15684, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3273925870000767, "nonpadding_tokens": 8566, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035095700002329977, "prefetch": {"data_wait_seconds": 0.004378409999731048, "consumed_batches": 440, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 445, "tokens": 40546692, "loss_tokens": 74606, "loss": 4.063684556201914, "lr": 0.00029998865975143355, "optimizer_lrs": [0.019999243983428904, 0.00029998865975143355], "gradient_norm": 0.5622069239616394, "seconds": 0.3121335390000013, "nonpadding_tokens_per_second": 239019.49223085472, "ranks": [{"compute_seconds": 0.3084940029999643, "nonpadding_tokens": 8049, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3093957419999924, "nonpadding_tokens": 9573, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30560664799997994, "nonpadding_tokens": 11153, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.309716797999954, "nonpadding_tokens": 4894, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3059376850000035, "nonpadding_tokens": 12345, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30534118100001706, "nonpadding_tokens": 17687, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.30628864400000566, "nonpadding_tokens": 5697, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3075825430000805, "nonpadding_tokens": 5208, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003992629999629571, "prefetch": {"data_wait_seconds": 0.0043875289997004074, "consumed_batches": 441, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 446, "tokens": 40652419, "loss_tokens": 105727, "loss": 3.7912412392293358, "lr": 0.00029998854274591023, "optimizer_lrs": [0.019999236183060685, 0.00029998854274591023], "gradient_norm": 1.4191126823425293, "seconds": 0.40841262899994035, "nonpadding_tokens_per_second": 258872.99386135154, "ranks": [{"compute_seconds": 0.34624046700002964, "nonpadding_tokens": 15093, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3675433679999287, "nonpadding_tokens": 8774, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3702564429999029, "nonpadding_tokens": 11285, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3707815779999919, "nonpadding_tokens": 15268, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36948691700001746, "nonpadding_tokens": 20084, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3718150830000013, "nonpadding_tokens": 6781, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3689410729999736, "nonpadding_tokens": 12809, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3696178120000013, "nonpadding_tokens": 15633, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003720089999887932, "prefetch": {"data_wait_seconds": 0.004397703999757141, "consumed_batches": 442, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 447, "tokens": 40715235, "loss_tokens": 62816, "loss": 3.296237473334819, "lr": 0.00029998847294458324, "optimizer_lrs": [0.019999231529638884, 0.00029998847294458324], "gradient_norm": 0.6064869165420532, "seconds": 0.3544528080000191, "nonpadding_tokens_per_second": 177219.64273448955, "ranks": [{"compute_seconds": 0.35124170999995386, "nonpadding_tokens": 13022, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3521177390000503, "nonpadding_tokens": 15112, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3479378600000018, "nonpadding_tokens": 8773, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3514985939999633, "nonpadding_tokens": 2166, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3488940710000179, "nonpadding_tokens": 7344, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34868674500000907, "nonpadding_tokens": 3199, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.34917072399991866, "nonpadding_tokens": 2772, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3491234919999897, "nonpadding_tokens": 10428, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036558299996158894, "prefetch": {"data_wait_seconds": 0.004406435999726455, "consumed_batches": 443, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 448, "tokens": 40807263, "loss_tokens": 92028, "loss": 3.628381104120485, "lr": 0.00029998837030003836, "optimizer_lrs": [0.019999224686669224, 0.00029998837030003836], "gradient_norm": 1.7048852443695068, "seconds": 0.34612884000000577, "nonpadding_tokens_per_second": 265877.8736842572, "ranks": [{"compute_seconds": 0.34148991200004275, "nonpadding_tokens": 13495, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31687878000002456, "nonpadding_tokens": 9029, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3131803949999039, "nonpadding_tokens": 15927, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3435761130000401, "nonpadding_tokens": 8943, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3138509809999732, "nonpadding_tokens": 9883, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31297829999994065, "nonpadding_tokens": 14764, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.31634772799998245, "nonpadding_tokens": 11028, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3156234540000469, "nonpadding_tokens": 8959, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003791719999526322, "prefetch": {"data_wait_seconds": 0.0044145729997353556, "consumed_batches": 444, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 449, "tokens": 40875973, "loss_tokens": 68710, "loss": 4.1229178794935235, "lr": 0.00029998829336685625, "optimizer_lrs": [0.019999219557790417, 0.00029998829336685625], "gradient_norm": 0.6135728359222412, "seconds": 0.37308060099996965, "nonpadding_tokens_per_second": 184169.3184149384, "ranks": [{"compute_seconds": 0.3385481020000043, "nonpadding_tokens": 3319, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3390599839999595, "nonpadding_tokens": 12087, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33489722199999505, "nonpadding_tokens": 14988, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37011304299994663, "nonpadding_tokens": 8704, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33789214099999754, "nonpadding_tokens": 11041, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36580106100007015, "nonpadding_tokens": 3744, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.335846317000005, "nonpadding_tokens": 9444, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33589157199992314, "nonpadding_tokens": 5383, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035577699998157186, "prefetch": {"data_wait_seconds": 0.004423881999741752, "consumed_batches": 445, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 450, "tokens": 40989021, "loss_tokens": 113048, "loss": 3.9630635327471517, "lr": 0.00029998816623741815, "optimizer_lrs": [0.019999211082494545, 0.00029998816623741815], "gradient_norm": 1.4268217086791992, "seconds": 0.3088188760000321, "nonpadding_tokens_per_second": 366065.70642394363, "ranks": [{"compute_seconds": 0.3058371710000074, "nonpadding_tokens": 14491, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3036258889999317, "nonpadding_tokens": 17573, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.304056773999946, "nonpadding_tokens": 11895, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30335319699997854, "nonpadding_tokens": 11869, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3059301499999947, "nonpadding_tokens": 17611, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3030058100000588, "nonpadding_tokens": 19855, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3040897870000663, "nonpadding_tokens": 10404, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30647799600001235, "nonpadding_tokens": 9350, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004073840000273776, "prefetch": {"data_wait_seconds": 0.004434753999817076, "consumed_batches": 446, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 451, "tokens": 41071169, "loss_tokens": 82148, "loss": 4.049692323611043, "lr": 0.00029998807342623314, "optimizer_lrs": [0.01999920489508221, 0.00029998807342623314], "gradient_norm": 2.9753177165985107, "seconds": 0.3749550199999021, "nonpadding_tokens_per_second": 219087.61216217733, "ranks": [{"compute_seconds": 0.3696462469999915, "nonpadding_tokens": 15865, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.37265781600001446, "nonpadding_tokens": 13854, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3698393649999616, "nonpadding_tokens": 9724, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36924006999993253, "nonpadding_tokens": 6407, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37183213599996634, "nonpadding_tokens": 15882, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36873423500003355, "nonpadding_tokens": 8799, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3703030619999481, "nonpadding_tokens": 3135, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3723599080000213, "nonpadding_tokens": 8482, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034845999994104204, "prefetch": {"data_wait_seconds": 0.004445158999828891, "consumed_batches": 447, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 452, "tokens": 41139713, "loss_tokens": 68544, "loss": 3.4237681726336366, "lr": 0.0002999879957074713, "optimizer_lrs": [0.019999199713831423, 0.0002999879957074713], "gradient_norm": 0.5718989968299866, "seconds": 0.43185289699999885, "nonpadding_tokens_per_second": 158720.71364152545, "ranks": [{"compute_seconds": 0.427784177000035, "nonpadding_tokens": 18520, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4289360730000453, "nonpadding_tokens": 8171, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4270594910000227, "nonpadding_tokens": 1975, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4294629650000843, "nonpadding_tokens": 10695, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4280185549999942, "nonpadding_tokens": 5303, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.429397602999984, "nonpadding_tokens": 4184, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4279461840000067, "nonpadding_tokens": 12759, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.42823787700001503, "nonpadding_tokens": 6937, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039047500001743174, "prefetch": {"data_wait_seconds": 0.004454600999793001, "consumed_batches": 448, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 453, "tokens": 41235384, "loss_tokens": 95671, "loss": 3.4957105601488436, "lr": 0.0002999878868087149, "optimizer_lrs": [0.019999192453914325, 0.0002999878868087149], "gradient_norm": 0.505231499671936, "seconds": 0.40785928899992996, "nonpadding_tokens_per_second": 234568.6431087179, "ranks": [{"compute_seconds": 0.4020026970000572, "nonpadding_tokens": 25383, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4054995440000084, "nonpadding_tokens": 20284, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4034860489999801, "nonpadding_tokens": 10214, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4012424570000803, "nonpadding_tokens": 6772, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4035096979999935, "nonpadding_tokens": 1705, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4015646820000711, "nonpadding_tokens": 13373, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4048257739999599, "nonpadding_tokens": 12106, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.40210767500002476, "nonpadding_tokens": 5834, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036821900005179486, "prefetch": {"data_wait_seconds": 0.0044638609997491585, "consumed_batches": 449, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 454, "tokens": 41305999, "loss_tokens": 70615, "loss": 3.472710737803583, "lr": 0.0002999878061148777, "optimizer_lrs": [0.01999918707432518, 0.0002999878061148777], "gradient_norm": 0.5170857310295105, "seconds": 0.4044812949999823, "nonpadding_tokens_per_second": 174581.62064083357, "ranks": [{"compute_seconds": 0.3996056299999964, "nonpadding_tokens": 11441, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40169591000005767, "nonpadding_tokens": 1915, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3981384740000067, "nonpadding_tokens": 7953, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4018320910000739, "nonpadding_tokens": 14279, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.40022585200006233, "nonpadding_tokens": 1683, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.40206476100001964, "nonpadding_tokens": 6730, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4006796639999948, "nonpadding_tokens": 18449, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.40128397699993457, "nonpadding_tokens": 8165, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000358431999984532, "prefetch": {"data_wait_seconds": 0.004473389999702704, "consumed_batches": 450, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 455, "tokens": 41391183, "loss_tokens": 85184, "loss": 3.9098147099220513, "lr": 0.0002999877084161414, "optimizer_lrs": [0.019999180561076094, 0.0002999877084161414], "gradient_norm": 0.5313789248466492, "seconds": 0.298748784000054, "nonpadding_tokens_per_second": 285135.88862000057, "ranks": [{"compute_seconds": 0.293111113000009, "nonpadding_tokens": 5323, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2961570569999594, "nonpadding_tokens": 12674, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2924449149999191, "nonpadding_tokens": 8878, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2963899170000559, "nonpadding_tokens": 10025, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2945050050000191, "nonpadding_tokens": 18655, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2924356830000079, "nonpadding_tokens": 6565, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2949994800000013, "nonpadding_tokens": 12210, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.29372618699994746, "nonpadding_tokens": 10854, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038216399991597427, "prefetch": {"data_wait_seconds": 0.004484780999746363, "consumed_batches": 451, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 456, "tokens": 41489770, "loss_tokens": 98587, "loss": 3.9340474656902025, "lr": 0.0002999875948586811, "optimizer_lrs": [0.019999172990578743, 0.0002999875948586811], "gradient_norm": 0.5130429267883301, "seconds": 0.3390663229999973, "nonpadding_tokens_per_second": 290760.2239223292, "ranks": [{"compute_seconds": 0.3238502450000169, "nonpadding_tokens": 11321, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3365685380000514, "nonpadding_tokens": 10432, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33490991100006795, "nonpadding_tokens": 12359, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3367610090000426, "nonpadding_tokens": 10149, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3359886659999347, "nonpadding_tokens": 15164, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3360098389999848, "nonpadding_tokens": 2390, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.33534513099994, "nonpadding_tokens": 18093, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3354568060000247, "nonpadding_tokens": 18679, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003468689999408525, "prefetch": {"data_wait_seconds": 0.004494769999837445, "consumed_batches": 452, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 457, "tokens": 41584265, "loss_tokens": 94495, "loss": 3.768959997883486, "lr": 0.00029998748552452503, "optimizer_lrs": [0.019999165701635003, 0.00029998748552452503], "gradient_norm": 0.4524836838245392, "seconds": 0.3943018549999806, "nonpadding_tokens_per_second": 239651.42137108295, "ranks": [{"compute_seconds": 0.38917629800005216, "nonpadding_tokens": 24807, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39184287800003403, "nonpadding_tokens": 19267, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3884139659999164, "nonpadding_tokens": 12449, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38723157399999764, "nonpadding_tokens": 14889, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39137866800001575, "nonpadding_tokens": 15410, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3919316249999838, "nonpadding_tokens": 3274, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.39186541100002614, "nonpadding_tokens": 957, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.38916397599996344, "nonpadding_tokens": 3442, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003977850000183025, "prefetch": {"data_wait_seconds": 0.0045047179999073705, "consumed_batches": 453, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 458, "tokens": 41704364, "loss_tokens": 120099, "loss": 3.8361474179635136, "lr": 0.0002999873458733608, "optimizer_lrs": [0.019999156391557388, 0.0002999873458733608], "gradient_norm": 0.9893535375595093, "seconds": 0.34690913999997974, "nonpadding_tokens_per_second": 346197.2780538645, "ranks": [{"compute_seconds": 0.34219938100000036, "nonpadding_tokens": 10946, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34041412299995955, "nonpadding_tokens": 21913, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3432196620000241, "nonpadding_tokens": 19102, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34451837599999635, "nonpadding_tokens": 11437, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34297998900001403, "nonpadding_tokens": 13837, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3402094699999907, "nonpadding_tokens": 2062, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.34066448900000523, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3434539179999092, "nonpadding_tokens": 8034, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003607109999848035, "prefetch": {"data_wait_seconds": 0.004514316999802759, "consumed_batches": 454, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 459, "tokens": 41791788, "loss_tokens": 87424, "loss": 3.463244503797584, "lr": 0.00029998724372939413, "optimizer_lrs": [0.01999914958195961, 0.00029998724372939413], "gradient_norm": 0.5418302416801453, "seconds": 0.39733388600006947, "nonpadding_tokens_per_second": 220026.53959391903, "ranks": [{"compute_seconds": 0.39496531899999354, "nonpadding_tokens": 22484, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39477142399994136, "nonpadding_tokens": 17997, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.39119803900007355, "nonpadding_tokens": 16552, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38987696699996377, "nonpadding_tokens": 1829, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39088372699995944, "nonpadding_tokens": 2846, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3900408839999727, "nonpadding_tokens": 10141, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3938684409999951, "nonpadding_tokens": 12816, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3927960419999863, "nonpadding_tokens": 2759, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003768530000343162, "prefetch": {"data_wait_seconds": 0.0045244339997907446, "consumed_batches": 455, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 460, "tokens": 41874380, "loss_tokens": 82592, "loss": 3.920377427595893, "lr": 0.0002999871468538476, "optimizer_lrs": [0.01999914312358984, 0.0002999871468538476], "gradient_norm": 0.5622161030769348, "seconds": 0.3123186699999678, "nonpadding_tokens_per_second": 264447.84745019727, "ranks": [{"compute_seconds": 0.30750398199995743, "nonpadding_tokens": 11801, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.305365337000012, "nonpadding_tokens": 7995, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30660653500001445, "nonpadding_tokens": 9356, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3099599039999248, "nonpadding_tokens": 3660, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3054738710000038, "nonpadding_tokens": 5024, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.306174685999963, "nonpadding_tokens": 12830, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3090262080000912, "nonpadding_tokens": 17183, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3080607560000317, "nonpadding_tokens": 14743, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003577549999818075, "prefetch": {"data_wait_seconds": 0.004534447999844815, "consumed_batches": 456, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 461, "tokens": 41957914, "loss_tokens": 83534, "loss": 3.6641128163382577, "lr": 0.000299987048500652, "optimizer_lrs": [0.019999136566710134, 0.000299987048500652], "gradient_norm": 0.48965561389923096, "seconds": 0.41272433500000716, "nonpadding_tokens_per_second": 202396.59481188224, "ranks": [{"compute_seconds": 0.40704947699998684, "nonpadding_tokens": 11003, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.39187665899999047, "nonpadding_tokens": 4870, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.38820033900003637, "nonpadding_tokens": 10384, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.39209743799995067, "nonpadding_tokens": 17551, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4093601829999898, "nonpadding_tokens": 13483, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.38787130999992314, "nonpadding_tokens": 1539, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3890491799999154, "nonpadding_tokens": 10681, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3883474839999508, "nonpadding_tokens": 14023, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003861320000169144, "prefetch": {"data_wait_seconds": 0.0045447939999121445, "consumed_batches": 457, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 462, "tokens": 42043556, "loss_tokens": 85642, "loss": 3.298024406833096, "lr": 0.00029998694727633005, "optimizer_lrs": [0.019999129818422005, 0.00029998694727633005], "gradient_norm": 0.45349302887916565, "seconds": 0.3286746869999888, "nonpadding_tokens_per_second": 260567.67797272722, "ranks": [{"compute_seconds": 0.3239600320000591, "nonpadding_tokens": 5438, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3260688340000115, "nonpadding_tokens": 6532, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32213599199997134, "nonpadding_tokens": 9729, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32219091900003605, "nonpadding_tokens": 12223, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3243537409999817, "nonpadding_tokens": 17250, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3219081599999072, "nonpadding_tokens": 9511, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3250616700000819, "nonpadding_tokens": 9483, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3248143440000604, "nonpadding_tokens": 15476, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0006236369999896851, "prefetch": {"data_wait_seconds": 0.004555688999971608, "consumed_batches": 458, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 463, "tokens": 42130912, "loss_tokens": 87356, "loss": 3.5753225307935343, "lr": 0.0002999868436202342, "optimizer_lrs": [0.019999122908015615, 0.0002999868436202342], "gradient_norm": 0.5682690739631653, "seconds": 0.3093371489999299, "nonpadding_tokens_per_second": 282397.3786608468, "ranks": [{"compute_seconds": 0.3043351910000638, "nonpadding_tokens": 10727, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30681546400001025, "nonpadding_tokens": 3440, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3033667949999881, "nonpadding_tokens": 6360, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30692697099993893, "nonpadding_tokens": 5658, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3052184739999575, "nonpadding_tokens": 18634, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30256876299995383, "nonpadding_tokens": 18469, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.302651663000006, "nonpadding_tokens": 9648, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30602988399994047, "nonpadding_tokens": 14420, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003896239999221507, "prefetch": {"data_wait_seconds": 0.00456663500006016, "consumed_batches": 459, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 464, "tokens": 42211877, "loss_tokens": 80965, "loss": 3.9047497375409126, "lr": 0.00029998674718161175, "optimizer_lrs": [0.01999911647877412, 0.00029998674718161175], "gradient_norm": 0.6688896417617798, "seconds": 0.30464795199998207, "nonpadding_tokens_per_second": 265765.7780676785, "ranks": [{"compute_seconds": 0.2512719850000167, "nonpadding_tokens": 4507, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.25903210800004217, "nonpadding_tokens": 5934, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.25556092900001204, "nonpadding_tokens": 7738, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2591942689999769, "nonpadding_tokens": 8015, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.25777065800002674, "nonpadding_tokens": 9936, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.25860904800003937, "nonpadding_tokens": 11951, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.25768344599998727, "nonpadding_tokens": 16544, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2560805949999576, "nonpadding_tokens": 16340, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.02731261799999629, "prefetch": {"data_wait_seconds": 0.004575954000074489, "consumed_batches": 460, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 465, "tokens": 42314165, "loss_tokens": 102288, "loss": 3.9806398355623336, "lr": 0.0002999866248413701, "optimizer_lrs": [0.019999108322758006, 0.0002999866248413701], "gradient_norm": 0.9156559705734253, "seconds": 0.4160982829999966, "nonpadding_tokens_per_second": 245826.53709244175, "ranks": [{"compute_seconds": 0.4132076879999431, "nonpadding_tokens": 17090, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.41376442299997507, "nonpadding_tokens": 13168, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.41016216499997427, "nonpadding_tokens": 9215, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.40971861299999546, "nonpadding_tokens": 11386, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.41268709699988904, "nonpadding_tokens": 12529, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.41358295099996667, "nonpadding_tokens": 17925, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4127563150000242, "nonpadding_tokens": 8094, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4104200570000103, "nonpadding_tokens": 12881, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003718910000998221, "prefetch": {"data_wait_seconds": 0.0045868900000414214, "consumed_batches": 461, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 466, "tokens": 42436371, "loss_tokens": 122206, "loss": 3.932091611704826, "lr": 0.0002999864779415779, "optimizer_lrs": [0.019999098529438528, 0.0002999864779415779], "gradient_norm": 0.6317479610443115, "seconds": 0.35420124200004466, "nonpadding_tokens_per_second": 345018.5530404904, "ranks": [{"compute_seconds": 0.3509797479999861, "nonpadding_tokens": 18326, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.347237990999929, "nonpadding_tokens": 16662, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34814178499993886, "nonpadding_tokens": 24982, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.35179822400004923, "nonpadding_tokens": 22920, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3504562440000427, "nonpadding_tokens": 6041, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34798589499996524, "nonpadding_tokens": 7754, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.34962110799995116, "nonpadding_tokens": 15338, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35057703199993284, "nonpadding_tokens": 10183, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003666999999722975, "prefetch": {"data_wait_seconds": 0.0045985289999634915, "consumed_batches": 462, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 467, "tokens": 42504187, "loss_tokens": 67816, "loss": 3.7500078336970626, "lr": 0.0002999863960760778, "optimizer_lrs": [0.01999909307173852, 0.0002999863960760778], "gradient_norm": 0.5481826663017273, "seconds": 0.891047993999905, "nonpadding_tokens_per_second": 76108.13385660035, "ranks": [{"compute_seconds": 0.8851625610000156, "nonpadding_tokens": 3681, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.8884315749999132, "nonpadding_tokens": 21221, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.8876512599999842, "nonpadding_tokens": 7080, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.8841038680000111, "nonpadding_tokens": 2801, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.8869722510000884, "nonpadding_tokens": 11638, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.8884698339999204, "nonpadding_tokens": 7735, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.8851104550000173, "nonpadding_tokens": 7773, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.8871001500000375, "nonpadding_tokens": 5887, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000411181000004035, "prefetch": {"data_wait_seconds": 0.004608253000014884, "consumed_batches": 463, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 468, "tokens": 42589848, "loss_tokens": 85661, "loss": 3.626845939225552, "lr": 0.0002999862923154892, "optimizer_lrs": [0.01999908615436595, 0.0002999862923154892], "gradient_norm": 0.5516275763511658, "seconds": 0.26397375099998044, "nonpadding_tokens_per_second": 324505.7498160351, "ranks": [{"compute_seconds": 0.18215498899996874, "nonpadding_tokens": 9779, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.18398146099991664, "nonpadding_tokens": 10035, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.18043969499990453, "nonpadding_tokens": 5012, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.17985747900002025, "nonpadding_tokens": 9936, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.18237463599996317, "nonpadding_tokens": 15624, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.18435424500000863, "nonpadding_tokens": 11679, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.18263744199998655, "nonpadding_tokens": 17187, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.17978623900000912, "nonpadding_tokens": 6409, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.008536374999948748, "prefetch": {"data_wait_seconds": 0.004617425000105868, "consumed_batches": 464, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 469, "tokens": 42704669, "loss_tokens": 114821, "loss": 3.886994212731121, "lr": 0.00029998615261529304, "optimizer_lrs": [0.019999076841019536, 0.00029998615261529304], "gradient_norm": 0.5092173218727112, "seconds": 0.24951042099996812, "nonpadding_tokens_per_second": 460185.1880167148, "ranks": [{"compute_seconds": 0.24639928600004168, "nonpadding_tokens": 18111, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.24685106699996595, "nonpadding_tokens": 19730, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2428347659999872, "nonpadding_tokens": 20840, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.24646760600001016, "nonpadding_tokens": 9502, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.24523021300001346, "nonpadding_tokens": 12412, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.24698791400010123, "nonpadding_tokens": 11341, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2431978590000199, "nonpadding_tokens": 12620, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2454090679999581, "nonpadding_tokens": 10265, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004336060000014186, "prefetch": {"data_wait_seconds": 0.004628378000120392, "consumed_batches": 465, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 470, "tokens": 42778510, "loss_tokens": 73841, "loss": 3.2806291558890046, "lr": 0.0002999860624003839, "optimizer_lrs": [0.01999907082669226, 0.0002999860624003839], "gradient_norm": 0.6388702988624573, "seconds": 0.2785785600000281, "nonpadding_tokens_per_second": 265063.47078537755, "ranks": [{"compute_seconds": 0.27253168099991854, "nonpadding_tokens": 5869, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.27593127900001946, "nonpadding_tokens": 15372, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.274670690999983, "nonpadding_tokens": 13126, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.27624813500005985, "nonpadding_tokens": 9019, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2724987139999939, "nonpadding_tokens": 7331, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2719329399999424, "nonpadding_tokens": 2758, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2717463339999995, "nonpadding_tokens": 12237, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.27253841099991405, "nonpadding_tokens": 8129, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037258099996506644, "prefetch": {"data_wait_seconds": 0.004637928000079228, "consumed_batches": 466, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 471, "tokens": 42919631, "loss_tokens": 141121, "loss": 3.6327745781988505, "lr": 0.0002999858891716336, "optimizer_lrs": [0.019999059278108907, 0.0002999858891716336], "gradient_norm": 1.0261842012405396, "seconds": 0.40660320999995747, "nonpadding_tokens_per_second": 347073.01007292775, "ranks": [{"compute_seconds": 0.40118338599995695, "nonpadding_tokens": 17636, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40026278599998477, "nonpadding_tokens": 17717, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4010190669999929, "nonpadding_tokens": 19584, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.40368134999994254, "nonpadding_tokens": 25703, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4026404269999375, "nonpadding_tokens": 11531, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4042653459999883, "nonpadding_tokens": 16924, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4027869220000184, "nonpadding_tokens": 12557, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.403486670999996, "nonpadding_tokens": 19469, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00034850299994104716, "prefetch": {"data_wait_seconds": 0.004647046000059163, "consumed_batches": 467, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 472, "tokens": 43053194, "loss_tokens": 133563, "loss": 3.7060167767270875, "lr": 0.0002999857242350596, "optimizer_lrs": [0.019999048282337305, 0.0002999857242350596], "gradient_norm": 1.0058231353759766, "seconds": 0.3131040020000455, "nonpadding_tokens_per_second": 426577.10903350444, "ranks": [{"compute_seconds": 0.3091716860000133, "nonpadding_tokens": 12585, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3059311130000424, "nonpadding_tokens": 17115, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30929731300000185, "nonpadding_tokens": 12233, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3108362790000001, "nonpadding_tokens": 26966, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3080111320000469, "nonpadding_tokens": 10521, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30624934100001155, "nonpadding_tokens": 18662, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3070474019999665, "nonpadding_tokens": 15361, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30913435700006175, "nonpadding_tokens": 20120, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038548899999568675, "prefetch": {"data_wait_seconds": 0.004656776000160789, "consumed_batches": 468, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 473, "tokens": 43196723, "loss_tokens": 143529, "loss": 3.6554507277274975, "lr": 0.0002999855459232756, "optimizer_lrs": [0.019999036394885043, 0.0002999855459232756], "gradient_norm": 1.1107754707336426, "seconds": 0.3392736189999823, "nonpadding_tokens_per_second": 423047.92345203675, "ranks": [{"compute_seconds": 0.30405731099995137, "nonpadding_tokens": 17617, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.323543250000057, "nonpadding_tokens": 29643, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31965096999999787, "nonpadding_tokens": 16402, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3191773030000604, "nonpadding_tokens": 18126, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.321166246999951, "nonpadding_tokens": 18123, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3190490370000134, "nonpadding_tokens": 14223, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.321595210000055, "nonpadding_tokens": 11395, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32010687199999666, "nonpadding_tokens": 18000, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0072867830000404865, "prefetch": {"data_wait_seconds": 0.004665220000106274, "consumed_batches": 469, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 474, "tokens": 43293367, "loss_tokens": 96644, "loss": 3.381457009747113, "lr": 0.0002999854252351474, "optimizer_lrs": [0.019999028349009827, 0.0002999854252351474], "gradient_norm": 0.9837653040885925, "seconds": 0.36575642800005426, "nonpadding_tokens_per_second": 264230.48947751004, "ranks": [{"compute_seconds": 0.36023690299998634, "nonpadding_tokens": 6556, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3609875979999515, "nonpadding_tokens": 22142, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3625521280000612, "nonpadding_tokens": 18324, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.361231513000007, "nonpadding_tokens": 18316, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36009810399991693, "nonpadding_tokens": 9311, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3633175539999911, "nonpadding_tokens": 7332, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.35990628000001834, "nonpadding_tokens": 9677, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35987700099997255, "nonpadding_tokens": 4986, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003808550000030664, "prefetch": {"data_wait_seconds": 0.004674280000131148, "consumed_batches": 470, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 475, "tokens": 43380615, "loss_tokens": 87248, "loss": 3.495392444525949, "lr": 0.0002999853158497171, "optimizer_lrs": [0.019999021056647807, 0.0002999853158497171], "gradient_norm": 0.5886768102645874, "seconds": 0.37277335299995684, "nonpadding_tokens_per_second": 234051.06426695178, "ranks": [{"compute_seconds": 0.3690198110000438, "nonpadding_tokens": 8806, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3704699040000605, "nonpadding_tokens": 18829, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36901799199995367, "nonpadding_tokens": 3192, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36980079599993587, "nonpadding_tokens": 10675, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3664337049999631, "nonpadding_tokens": 14835, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36700151200000164, "nonpadding_tokens": 5271, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.37024507400008133, "nonpadding_tokens": 14809, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3688894200000732, "nonpadding_tokens": 10831, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003748550000182149, "prefetch": {"data_wait_seconds": 0.004684646000100656, "consumed_batches": 471, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 476, "tokens": 43437960, "loss_tokens": 57345, "loss": 3.6876051748190775, "lr": 0.00029998524373186304, "optimizer_lrs": [0.01999901624879087, 0.00029998524373186304], "gradient_norm": 0.6262420415878296, "seconds": 0.39614467199999126, "nonpadding_tokens_per_second": 144757.721239783, "ranks": [{"compute_seconds": 0.39085000400007175, "nonpadding_tokens": 10313, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.393804197999998, "nonpadding_tokens": 2587, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.39022714500003985, "nonpadding_tokens": 1167, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.39383453999994344, "nonpadding_tokens": 2973, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39312000600000374, "nonpadding_tokens": 6582, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3900140120000515, "nonpadding_tokens": 5443, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3909666209999614, "nonpadding_tokens": 13495, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.392575119000071, "nonpadding_tokens": 14785, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003935590000310185, "prefetch": {"data_wait_seconds": 0.004694635000078051, "consumed_batches": 472, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 477, "tokens": 43507352, "loss_tokens": 69392, "loss": 3.558582491497579, "lr": 0.000299985156227321, "optimizer_lrs": [0.019999010415154735, 0.000299985156227321], "gradient_norm": 0.9442265629768372, "seconds": 0.5023432049999883, "nonpadding_tokens_per_second": 138136.635091942, "ranks": [{"compute_seconds": 0.49938268799996877, "nonpadding_tokens": 10074, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4999858519998952, "nonpadding_tokens": 2772, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.49722718699990764, "nonpadding_tokens": 13722, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.49711185600006047, "nonpadding_tokens": 4583, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.49759613399999125, "nonpadding_tokens": 18529, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.49707082800000535, "nonpadding_tokens": 8599, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.49972100999991653, "nonpadding_tokens": 5781, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.49962642599996343, "nonpadding_tokens": 5332, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039473599997563724, "prefetch": {"data_wait_seconds": 0.00470482700006869, "consumed_batches": 473, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 478, "tokens": 43598478, "loss_tokens": 91126, "loss": 3.5883796885630885, "lr": 0.00029998504092293687, "optimizer_lrs": [0.019999002728195794, 0.00029998504092293687], "gradient_norm": 0.5022678971290588, "seconds": 0.3299186069999678, "nonpadding_tokens_per_second": 276207.51926855976, "ranks": [{"compute_seconds": 0.3267983829999821, "nonpadding_tokens": 7911, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32712966799999776, "nonpadding_tokens": 11468, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.32226128900003914, "nonpadding_tokens": 10263, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32250119100001484, "nonpadding_tokens": 11526, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3235283620000473, "nonpadding_tokens": 8790, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32746472300004825, "nonpadding_tokens": 19438, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3254650699999502, "nonpadding_tokens": 13852, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32552190299998074, "nonpadding_tokens": 7878, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004050560000905534, "prefetch": {"data_wait_seconds": 0.004713150000156929, "consumed_batches": 474, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 479, "tokens": 43695470, "loss_tokens": 96992, "loss": 3.880161830357143, "lr": 0.0002999849177060564, "optimizer_lrs": [0.019998994513737094, 0.0002999849177060564], "gradient_norm": 0.437991738319397, "seconds": 0.35424919000001864, "nonpadding_tokens_per_second": 273795.9683125737, "ranks": [{"compute_seconds": 0.3479048750000402, "nonpadding_tokens": 14638, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35174581299997953, "nonpadding_tokens": 2608, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3493920399999979, "nonpadding_tokens": 7967, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3472372770000902, "nonpadding_tokens": 11297, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3506892619999462, "nonpadding_tokens": 21378, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3519210470000189, "nonpadding_tokens": 16049, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.34831531699990137, "nonpadding_tokens": 6975, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35072504599997956, "nonpadding_tokens": 16080, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037766800005556433, "prefetch": {"data_wait_seconds": 0.00472429500018734, "consumed_batches": 475, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 480, "tokens": 43815899, "loss_tokens": 120429, "loss": 3.6068342135199996, "lr": 0.00029998476401195455, "optimizer_lrs": [0.01999898426746364, 0.00029998476401195455], "gradient_norm": 0.5020575523376465, "seconds": 0.39758165000000645, "nonpadding_tokens_per_second": 302903.818624421, "ranks": [{"compute_seconds": 0.3676984189999075, "nonpadding_tokens": 17565, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3764022450000084, "nonpadding_tokens": 13967, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3729869349999717, "nonpadding_tokens": 21178, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3736048480000136, "nonpadding_tokens": 6136, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3759029489999648, "nonpadding_tokens": 22545, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3738983809998899, "nonpadding_tokens": 10296, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.373687077999989, "nonpadding_tokens": 10760, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37594745199999124, "nonpadding_tokens": 17982, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0005109280000397121, "prefetch": {"data_wait_seconds": 0.004732950000175151, "consumed_batches": 476, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 481, "tokens": 43921778, "loss_tokens": 105879, "loss": 3.8102064030638747, "lr": 0.0002999846282432939, "optimizer_lrs": [0.019998975216219596, 0.0002999846282432939], "gradient_norm": 0.4794076085090637, "seconds": 0.35154459000000315, "nonpadding_tokens_per_second": 301182.2767632381, "ranks": [{"compute_seconds": 0.3482445679999273, "nonpadding_tokens": 10261, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34836534900000515, "nonpadding_tokens": 15277, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3450959879999118, "nonpadding_tokens": 9528, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34905255399996804, "nonpadding_tokens": 11065, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34751389000007293, "nonpadding_tokens": 14068, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.34855965899998864, "nonpadding_tokens": 14655, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.34491545500009124, "nonpadding_tokens": 21259, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34745748799991816, "nonpadding_tokens": 9766, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004237220000504749, "prefetch": {"data_wait_seconds": 0.004743445000258362, "consumed_batches": 477, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 482, "tokens": 44007348, "loss_tokens": 85570, "loss": 3.6085660862451796, "lr": 0.0002999845180768499, "optimizer_lrs": [0.019998967871789997, 0.0002999845180768499], "gradient_norm": 0.6954013705253601, "seconds": 0.2806709750000209, "nonpadding_tokens_per_second": 304876.55519062356, "ranks": [{"compute_seconds": 0.24805025099999511, "nonpadding_tokens": 7080, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.24844825599996057, "nonpadding_tokens": 12630, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2756216170000698, "nonpadding_tokens": 8921, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.24370553200003542, "nonpadding_tokens": 12893, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.24656872599996404, "nonpadding_tokens": 6412, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.24542545199994947, "nonpadding_tokens": 11815, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2476274260000082, "nonpadding_tokens": 19423, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2774490799999967, "nonpadding_tokens": 6396, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004234919999817066, "prefetch": {"data_wait_seconds": 0.004753522000214616, "consumed_batches": 478, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 483, "tokens": 44074489, "loss_tokens": 67141, "loss": 3.7183781147138113, "lr": 0.0002999844313612883, "optimizer_lrs": [0.019998962090752557, 0.0002999844313612883], "gradient_norm": 0.6539314389228821, "seconds": 0.39409678300000905, "nonpadding_tokens_per_second": 170366.7801825179, "ranks": [{"compute_seconds": 0.3912175880000177, "nonpadding_tokens": 11749, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3824865699999691, "nonpadding_tokens": 8629, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3785183850000067, "nonpadding_tokens": 3087, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3785746689999314, "nonpadding_tokens": 10424, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37932238500002313, "nonpadding_tokens": 15004, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.38369483899998613, "nonpadding_tokens": 5888, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3889007909999691, "nonpadding_tokens": 8357, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.38128171000005295, "nonpadding_tokens": 4003, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003944410000258358, "prefetch": {"data_wait_seconds": 0.004762562000223625, "consumed_batches": 479, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 484, "tokens": 44176446, "loss_tokens": 101957, "loss": 3.6603194238747707, "lr": 0.00029998429921625387, "optimizer_lrs": [0.019998953281083594, 0.00029998429921625387], "gradient_norm": 0.9550926089286804, "seconds": 0.4321179989999564, "nonpadding_tokens_per_second": 235947.12609971678, "ranks": [{"compute_seconds": 0.40728833200000736, "nonpadding_tokens": 23823, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.40414733199997954, "nonpadding_tokens": 13767, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.40486260199998014, "nonpadding_tokens": 13505, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.40875752800002374, "nonpadding_tokens": 10711, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.40501753399996687, "nonpadding_tokens": 7392, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.40455392199999096, "nonpadding_tokens": 12348, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4289359219999369, "nonpadding_tokens": 14570, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4287866629999826, "nonpadding_tokens": 5841, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0005962620000445895, "prefetch": {"data_wait_seconds": 0.004774385000246184, "consumed_batches": 480, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 485, "tokens": 44256211, "loss_tokens": 79765, "loss": 4.107603585532502, "lr": 0.000299984195444632, "optimizer_lrs": [0.019998946362975466, 0.000299984195444632], "gradient_norm": 0.47279056906700134, "seconds": 0.3338595190000433, "nonpadding_tokens_per_second": 238917.85454824686, "ranks": [{"compute_seconds": 0.33060694599998897, "nonpadding_tokens": 5876, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32888637199994264, "nonpadding_tokens": 9469, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3295624349999571, "nonpadding_tokens": 4341, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32893978399999924, "nonpadding_tokens": 10201, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3279310789999954, "nonpadding_tokens": 1112, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.331468043999962, "nonpadding_tokens": 17164, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.32778108700006214, "nonpadding_tokens": 19905, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3279552470000908, "nonpadding_tokens": 11697, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003684480000174517, "prefetch": {"data_wait_seconds": 0.004785054000194577, "consumed_batches": 481, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 486, "tokens": 44306679, "loss_tokens": 50468, "loss": 3.667989183740192, "lr": 0.00029998412961090116, "optimizer_lrs": [0.01999894197406008, 0.00029998412961090116], "gradient_norm": 0.6192940473556519, "seconds": 0.3221030329999621, "nonpadding_tokens_per_second": 156682.78416988993, "ranks": [{"compute_seconds": 0.3167070560000411, "nonpadding_tokens": 1037, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31977289400003883, "nonpadding_tokens": 4334, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31903076300000066, "nonpadding_tokens": 10521, "peak_allocated_bytes": 14991537664, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3158388340000329, "nonpadding_tokens": 5554, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31706881700006306, "nonpadding_tokens": 12956, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31595575300002565, "nonpadding_tokens": 6857, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.31710015700002714, "nonpadding_tokens": 3823, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3173965579999276, "nonpadding_tokens": 5386, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003851239999903555, "prefetch": {"data_wait_seconds": 0.004794834000222181, "consumed_batches": 482, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 487, "tokens": 44408672, "loss_tokens": 101993, "loss": 3.635615618228702, "lr": 0.0002999839961469602, "optimizer_lrs": [0.019998933076464016, 0.0002999839961469602], "gradient_norm": 0.4773867428302765, "seconds": 0.41421936599999754, "nonpadding_tokens_per_second": 246229.4338985604, "ranks": [{"compute_seconds": 0.40913950499998464, "nonpadding_tokens": 19366, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4106222540000317, "nonpadding_tokens": 12600, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4067448210000748, "nonpadding_tokens": 14250, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.40679988499994124, "nonpadding_tokens": 17946, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.40911922400005096, "nonpadding_tokens": 10781, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4117670440000438, "nonpadding_tokens": 14073, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.40800447799995254, "nonpadding_tokens": 11613, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4075125739999521, "nonpadding_tokens": 1364, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036951200002022233, "prefetch": {"data_wait_seconds": 0.004803226000262839, "consumed_batches": 483, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 488, "tokens": 44496475, "loss_tokens": 87803, "loss": 3.705849316082594, "lr": 0.0002999838808038833, "optimizer_lrs": [0.019998925386925555, 0.0002999838808038833], "gradient_norm": 0.465302973985672, "seconds": 0.34227336999992986, "nonpadding_tokens_per_second": 256528.86755407817, "ranks": [{"compute_seconds": 0.30911172499997974, "nonpadding_tokens": 6542, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3395776229999683, "nonpadding_tokens": 10779, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3362311610000006, "nonpadding_tokens": 20406, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3356527079999978, "nonpadding_tokens": 11957, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33769833600001675, "nonpadding_tokens": 16477, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3397508900000048, "nonpadding_tokens": 4557, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.337836091999975, "nonpadding_tokens": 8923, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3388560560000542, "nonpadding_tokens": 8162, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036234999993212114, "prefetch": {"data_wait_seconds": 0.004813093000279878, "consumed_batches": 484, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 489, "tokens": 44598275, "loss_tokens": 101800, "loss": 3.0658500122789785, "lr": 0.0002999837465551162, "optimizer_lrs": [0.019998916437007748, 0.0002999837465551162], "gradient_norm": 0.45111292600631714, "seconds": 0.3951045899999599, "nonpadding_tokens_per_second": 257653.29630822648, "ranks": [{"compute_seconds": 0.3925489530000732, "nonpadding_tokens": 24016, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.38848098500000106, "nonpadding_tokens": 11767, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3919769980000183, "nonpadding_tokens": 3832, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3909974569999122, "nonpadding_tokens": 19870, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39055212100004155, "nonpadding_tokens": 18424, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.38819049399990035, "nonpadding_tokens": 10130, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.39087497000002713, "nonpadding_tokens": 7838, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3908026730000529, "nonpadding_tokens": 5923, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00044931399997949484, "prefetch": {"data_wait_seconds": 0.0048246170002812505, "consumed_batches": 485, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 490, "tokens": 44685155, "loss_tokens": 86880, "loss": 3.7834700161141805, "lr": 0.0002999836315418122, "optimizer_lrs": [0.01999890876945415, 0.0002999836315418122], "gradient_norm": 0.4371969401836395, "seconds": 0.31945519900000363, "nonpadding_tokens_per_second": 271963.0178878354, "ranks": [{"compute_seconds": 0.2834774319999269, "nonpadding_tokens": 8105, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.28903204200003074, "nonpadding_tokens": 5313, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.28974257099991974, "nonpadding_tokens": 5924, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.29308139299996583, "nonpadding_tokens": 15893, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.29016124700001455, "nonpadding_tokens": 17007, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2946179650000431, "nonpadding_tokens": 8437, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2915688240000236, "nonpadding_tokens": 9706, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.29020124299995587, "nonpadding_tokens": 16495, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0007131600000320759, "prefetch": {"data_wait_seconds": 0.004834829000287755, "consumed_batches": 486, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 491, "tokens": 44795113, "loss_tokens": 109958, "loss": 3.5468866521762856, "lr": 0.00029998348539610046, "optimizer_lrs": [0.0199988990264067, 0.00029998348539610046], "gradient_norm": 0.6471233367919922, "seconds": 0.3696535340000082, "nonpadding_tokens_per_second": 297462.32589784346, "ranks": [{"compute_seconds": 0.36602435899999364, "nonpadding_tokens": 1491, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3672601170000007, "nonpadding_tokens": 16940, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3644597989999738, "nonpadding_tokens": 21367, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3632061010000598, "nonpadding_tokens": 13001, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3644896970000673, "nonpadding_tokens": 17758, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3633856150000838, "nonpadding_tokens": 15325, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3648154979999845, "nonpadding_tokens": 13362, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3669097659999352, "nonpadding_tokens": 10714, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038720999998531624, "prefetch": {"data_wait_seconds": 0.00486307500023031, "consumed_batches": 487, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 492, "tokens": 44883099, "loss_tokens": 87986, "loss": 3.278589420475985, "lr": 0.0002999833679856942, "optimizer_lrs": [0.01999889119904628, 0.0002999833679856942], "gradient_norm": 0.4663240909576416, "seconds": 0.363847236999959, "nonpadding_tokens_per_second": 241821.26742386093, "ranks": [{"compute_seconds": 0.3600919470000008, "nonpadding_tokens": 17292, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35682767800005877, "nonpadding_tokens": 4376, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.35759412200002316, "nonpadding_tokens": 12753, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36137420599993675, "nonpadding_tokens": 13418, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3599390570000196, "nonpadding_tokens": 9731, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36144470799990813, "nonpadding_tokens": 15058, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3601602539999931, "nonpadding_tokens": 9280, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3597733190000554, "nonpadding_tokens": 6078, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003826810000191472, "prefetch": {"data_wait_seconds": 0.00487232900024992, "consumed_batches": 488, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 493, "tokens": 44962745, "loss_tokens": 79646, "loss": 3.5283590826909075, "lr": 0.00029998326134576125, "optimizer_lrs": [0.01999888408971742, 0.00029998326134576125], "gradient_norm": 0.5007596611976624, "seconds": 0.32959512200000063, "nonpadding_tokens_per_second": 241647.9938073836, "ranks": [{"compute_seconds": 0.3269812459999457, "nonpadding_tokens": 6680, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32721412900002633, "nonpadding_tokens": 2834, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3233568109999396, "nonpadding_tokens": 8913, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3269510860000082, "nonpadding_tokens": 16291, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32214951899993594, "nonpadding_tokens": 13225, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32324892700000873, "nonpadding_tokens": 18707, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.325509927999974, "nonpadding_tokens": 1834, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3255246679999573, "nonpadding_tokens": 11162, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.000370390999933079, "prefetch": {"data_wait_seconds": 0.004882293000264326, "consumed_batches": 489, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 494, "tokens": 45047875, "loss_tokens": 85130, "loss": 3.4891720750616706, "lr": 0.00029998314698640723, "optimizer_lrs": [0.019998876465760484, 0.00029998314698640723], "gradient_norm": 0.4832099378108978, "seconds": 0.30975271099998736, "nonpadding_tokens_per_second": 274832.1385958862, "ranks": [{"compute_seconds": 0.3057915870000443, "nonpadding_tokens": 3769, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.30241365900008077, "nonpadding_tokens": 9722, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3037584960000004, "nonpadding_tokens": 14760, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3073649309999382, "nonpadding_tokens": 9871, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3055910440000389, "nonpadding_tokens": 9222, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30315562800001317, "nonpadding_tokens": 7501, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3036865059999627, "nonpadding_tokens": 22233, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3059377790000326, "nonpadding_tokens": 8052, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004173610000179906, "prefetch": {"data_wait_seconds": 0.0048935440001969255, "consumed_batches": 490, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 495, "tokens": 45134425, "loss_tokens": 86550, "loss": 3.458707394569613, "lr": 0.0002999830303204007, "optimizer_lrs": [0.019998868688026715, 0.0002999830303204007], "gradient_norm": 0.47304508090019226, "seconds": 0.48237914400010595, "nonpadding_tokens_per_second": 179423.18003694827, "ranks": [{"compute_seconds": 0.47727091500007646, "nonpadding_tokens": 11959, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.479996419000031, "nonpadding_tokens": 13299, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.47843776599995635, "nonpadding_tokens": 4645, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4750656019999724, "nonpadding_tokens": 12152, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.47835188299995934, "nonpadding_tokens": 9448, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4759108229999356, "nonpadding_tokens": 6864, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.47889634199998454, "nonpadding_tokens": 18148, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.47866927500001566, "nonpadding_tokens": 10035, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003666110000040135, "prefetch": {"data_wait_seconds": 0.004903438000155802, "consumed_batches": 491, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 496, "tokens": 45226271, "loss_tokens": 91846, "loss": 3.5690544498399497, "lr": 0.0002999829060755098, "optimizer_lrs": [0.019998860405033986, 0.0002999829060755098], "gradient_norm": 0.4656749963760376, "seconds": 0.3240683750000244, "nonpadding_tokens_per_second": 283415.4983496711, "ranks": [{"compute_seconds": 0.32015378100004455, "nonpadding_tokens": 1913, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.31725435599992124, "nonpadding_tokens": 14040, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31724759700000504, "nonpadding_tokens": 17602, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31718261400010306, "nonpadding_tokens": 9559, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3182510059999686, "nonpadding_tokens": 11202, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3216712290000032, "nonpadding_tokens": 4650, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.31952255100009097, "nonpadding_tokens": 18687, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3177812559999893, "nonpadding_tokens": 14193, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038119600003483356, "prefetch": {"data_wait_seconds": 0.004912640000156898, "consumed_batches": 492, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 497, "tokens": 45325812, "loss_tokens": 99541, "loss": 3.4099689449573543, "lr": 0.00029998277090950286, "optimizer_lrs": [0.019998851393966857, 0.00029998277090950286], "gradient_norm": 0.9186007380485535, "seconds": 0.3629629790000308, "nonpadding_tokens_per_second": 274245.60012769664, "ranks": [{"compute_seconds": 0.36058344200000647, "nonpadding_tokens": 10036, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35622162600009233, "nonpadding_tokens": 21703, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3565861179999956, "nonpadding_tokens": 12873, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3562345179999511, "nonpadding_tokens": 9050, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.35879062799995154, "nonpadding_tokens": 24465, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3566431909999892, "nonpadding_tokens": 2440, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.35761143100000936, "nonpadding_tokens": 2079, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.35894516999996995, "nonpadding_tokens": 16895, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004331559999855017, "prefetch": {"data_wait_seconds": 0.004921526000089216, "consumed_batches": 493, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 498, "tokens": 45404052, "loss_tokens": 78240, "loss": 3.800369456160532, "lr": 0.00029998266429437316, "optimizer_lrs": [0.019998844286291544, 0.00029998266429437316], "gradient_norm": 1.1099804639816284, "seconds": 0.32469187000003785, "nonpadding_tokens_per_second": 240966.92042209397, "ranks": [{"compute_seconds": 0.3212250020000056, "nonpadding_tokens": 3497, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3162968930000716, "nonpadding_tokens": 4809, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31760025200003383, "nonpadding_tokens": 5729, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3164373949999799, "nonpadding_tokens": 10390, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.319163967999998, "nonpadding_tokens": 8045, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32166285300002073, "nonpadding_tokens": 15647, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.31947621300002993, "nonpadding_tokens": 18515, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31756359799999245, "nonpadding_tokens": 11608, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004136040000730645, "prefetch": {"data_wait_seconds": 0.004935228000135794, "consumed_batches": 494, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 499, "tokens": 45539378, "loss_tokens": 135326, "loss": 3.271746098310746, "lr": 0.00029998247911373747, "optimizer_lrs": [0.019998831940915833, 0.00029998247911373747], "gradient_norm": 0.6861430406570435, "seconds": 0.2735985049999954, "nonpadding_tokens_per_second": 494615.27576695743, "ranks": [{"compute_seconds": 0.27052778099994157, "nonpadding_tokens": 5592, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.27103208099993026, "nonpadding_tokens": 10562, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2689659059999485, "nonpadding_tokens": 21661, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2661445100000037, "nonpadding_tokens": 18217, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.26992384699997274, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2668099339999799, "nonpadding_tokens": 26850, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.269504760000018, "nonpadding_tokens": 12860, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.26778907100003835, "nonpadding_tokens": 6816, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038414499999817053, "prefetch": {"data_wait_seconds": 0.004945081000187201, "consumed_batches": 495, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 500, "tokens": 45650831, "loss_tokens": 111453, "loss": 3.415171081083506, "lr": 0.00029998232586227506, "optimizer_lrs": [0.019998821724151673, 0.00029998232586227506], "gradient_norm": 0.4566454589366913, "seconds": 0.3138328489999367, "nonpadding_tokens_per_second": 355134.9081371099, "ranks": [{"compute_seconds": 0.31158578800000214, "nonpadding_tokens": 10099, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3077049339999576, "nonpadding_tokens": 12145, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30836182100006226, "nonpadding_tokens": 11897, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3080451789999188, "nonpadding_tokens": 20935, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30848022399993624, "nonpadding_tokens": 24704, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30774676399994405, "nonpadding_tokens": 5652, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3100966800000151, "nonpadding_tokens": 21663, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31097298800000317, "nonpadding_tokens": 4358, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036344000000099186, "prefetch": {"data_wait_seconds": 0.004953874000193537, "consumed_batches": 496, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 501, "tokens": 45727298, "loss_tokens": 76467, "loss": 3.014874063321433, "lr": 0.0002999822203317352, "optimizer_lrs": [0.019998814688782347, 0.0002999822203317352], "gradient_norm": 0.6116868853569031, "seconds": 0.32452947000001586, "nonpadding_tokens_per_second": 235624.21002935807, "ranks": [{"compute_seconds": 0.31995610699993904, "nonpadding_tokens": 3543, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32188879199998155, "nonpadding_tokens": 1257, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31952237400003014, "nonpadding_tokens": 6207, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3165370729999495, "nonpadding_tokens": 7435, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3186367240000436, "nonpadding_tokens": 13589, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3167142929999045, "nonpadding_tokens": 10147, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3197974030000523, "nonpadding_tokens": 17960, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3174891020000814, "nonpadding_tokens": 16329, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004172559999915393, "prefetch": {"data_wait_seconds": 0.0049643250001736305, "consumed_batches": 497, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 502, "tokens": 45812657, "loss_tokens": 85359, "loss": 3.640718538759826, "lr": 0.0002999821021585187, "optimizer_lrs": [0.019998806810567916, 0.0002999821021585187], "gradient_norm": 1.185575008392334, "seconds": 0.2850787869999749, "nonpadding_tokens_per_second": 299422.48912405927, "ranks": [{"compute_seconds": 0.2820968180000136, "nonpadding_tokens": 2493, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.28248462700003074, "nonpadding_tokens": 17977, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.27934963500001686, "nonpadding_tokens": 7921, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2826512859999184, "nonpadding_tokens": 13044, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2815857599999845, "nonpadding_tokens": 4522, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.27880145800008904, "nonpadding_tokens": 12172, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2820323369999187, "nonpadding_tokens": 16587, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2813516339999751, "nonpadding_tokens": 10643, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003927139999859719, "prefetch": {"data_wait_seconds": 0.00497603400015123, "consumed_batches": 498, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 503, "tokens": 45876966, "loss_tokens": 64309, "loss": 3.588922477802485, "lr": 0.0002999820128689525, "optimizer_lrs": [0.01999880085793017, 0.0002999820128689525], "gradient_norm": 0.5346903204917908, "seconds": 0.41761769599997933, "nonpadding_tokens_per_second": 153990.12210441194, "ranks": [{"compute_seconds": 0.41266243499990196, "nonpadding_tokens": 14314, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.41006846499999483, "nonpadding_tokens": 4006, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4104992879999827, "nonpadding_tokens": 2662, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4140599589999283, "nonpadding_tokens": 7629, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.412616799000034, "nonpadding_tokens": 2498, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.41490983599999254, "nonpadding_tokens": 11490, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.41121253999995133, "nonpadding_tokens": 2684, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4130755850000014, "nonpadding_tokens": 19026, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004012169999896287, "prefetch": {"data_wait_seconds": 0.0049860950001630044, "consumed_batches": 499, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 504, "tokens": 45948463, "loss_tokens": 71497, "loss": 3.728673930374701, "lr": 0.00029998191333844567, "optimizer_lrs": [0.019998794222563045, 0.00029998191333844567], "gradient_norm": 0.4927213191986084, "seconds": 0.32036430499999824, "nonpadding_tokens_per_second": 223174.05180330685, "ranks": [{"compute_seconds": 0.31587719900005595, "nonpadding_tokens": 10562, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3154675999999199, "nonpadding_tokens": 5818, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.31460723600002893, "nonpadding_tokens": 5908, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31802406099996006, "nonpadding_tokens": 12760, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3172547259999874, "nonpadding_tokens": 6127, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31489901399993414, "nonpadding_tokens": 3428, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3176746340000136, "nonpadding_tokens": 23042, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3174301919999607, "nonpadding_tokens": 3852, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003791450000107943, "prefetch": {"data_wait_seconds": 0.0049961970000822475, "consumed_batches": 500, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 505, "tokens": 46037151, "loss_tokens": 88688, "loss": 3.376259330461844, "lr": 0.0002999817894948908, "optimizer_lrs": [0.019998785966326055, 0.0002999817894948908], "gradient_norm": 1.0218956470489502, "seconds": 0.32384404000003997, "nonpadding_tokens_per_second": 273860.21987617575, "ranks": [{"compute_seconds": 0.3183925769999405, "nonpadding_tokens": 11328, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3213254810000308, "nonpadding_tokens": 6212, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3167042569999694, "nonpadding_tokens": 18720, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3173935239999537, "nonpadding_tokens": 11259, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.319973535000031, "nonpadding_tokens": 18983, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31675073099995643, "nonpadding_tokens": 10093, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.31808194699999603, "nonpadding_tokens": 5145, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3197358490000397, "nonpadding_tokens": 6948, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040293500001098437, "prefetch": {"data_wait_seconds": 0.005006884000067657, "consumed_batches": 501, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 506, "tokens": 46090580, "loss_tokens": 53429, "loss": 3.1916857535233674, "lr": 0.00029998171468291376, "optimizer_lrs": [0.01999878097886092, 0.00029998171468291376], "gradient_norm": 0.5298799872398376, "seconds": 0.33281517599994004, "nonpadding_tokens_per_second": 160536.5495713141, "ranks": [{"compute_seconds": 0.3263794949999692, "nonpadding_tokens": 9720, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32985338999992564, "nonpadding_tokens": 1383, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3261310399999502, "nonpadding_tokens": 4741, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.32993307399999594, "nonpadding_tokens": 6148, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32846957999993265, "nonpadding_tokens": 9193, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.33007383200003915, "nonpadding_tokens": 5028, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3280261200000041, "nonpadding_tokens": 7847, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.32847788399999445, "nonpadding_tokens": 9369, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003969719999759036, "prefetch": {"data_wait_seconds": 0.00501703500003714, "consumed_batches": 502, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 507, "tokens": 46198889, "loss_tokens": 108309, "loss": 3.5359316284888607, "lr": 0.0002999815625567477, "optimizer_lrs": [0.019998770837116514, 0.0002999815625567477], "gradient_norm": 0.5202324390411377, "seconds": 0.300752043999978, "nonpadding_tokens_per_second": 360127.2282625215, "ranks": [{"compute_seconds": 0.2974410619999617, "nonpadding_tokens": 17787, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.294354568000017, "nonpadding_tokens": 13605, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2947999779999009, "nonpadding_tokens": 9823, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2984560889999557, "nonpadding_tokens": 11580, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.29439298499994493, "nonpadding_tokens": 21355, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.29469750099997327, "nonpadding_tokens": 17776, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.29734085099994445, "nonpadding_tokens": 6157, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2974701330000471, "nonpadding_tokens": 10226, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039736899998388253, "prefetch": {"data_wait_seconds": 0.005027170000062142, "consumed_batches": 503, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 508, "tokens": 46249657, "loss_tokens": 50768, "loss": 3.247726791482824, "lr": 0.00029998149103328527, "optimizer_lrs": [0.019998766068885687, 0.00029998149103328527], "gradient_norm": 0.6299394965171814, "seconds": 0.2605416180000475, "nonpadding_tokens_per_second": 194855.62571424097, "ranks": [{"compute_seconds": 0.257013640000082, "nonpadding_tokens": 1065, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2581085490000987, "nonpadding_tokens": 9289, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2575902789999418, "nonpadding_tokens": 10802, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2582686800000147, "nonpadding_tokens": 12162, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.25496276800004125, "nonpadding_tokens": 7526, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.25429524100002254, "nonpadding_tokens": 4385, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.257192023000016, "nonpadding_tokens": 2276, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2571056120000321, "nonpadding_tokens": 3263, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003754979999257557, "prefetch": {"data_wait_seconds": 0.0050370980001162025, "consumed_batches": 504, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 509, "tokens": 46342292, "loss_tokens": 92635, "loss": 3.4097088033680576, "lr": 0.0002999813601695581, "optimizer_lrs": [0.019998757344637207, 0.0002999813601695581], "gradient_norm": 0.6710785627365112, "seconds": 0.3127873679999311, "nonpadding_tokens_per_second": 296159.6582123495, "ranks": [{"compute_seconds": 0.3084992430000284, "nonpadding_tokens": 11175, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3104920179999908, "nonpadding_tokens": 15825, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30942888200002017, "nonpadding_tokens": 8152, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3063879999999699, "nonpadding_tokens": 22947, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3094232579999243, "nonpadding_tokens": 6120, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.30671678199996677, "nonpadding_tokens": 10578, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3094506300000148, "nonpadding_tokens": 7919, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3076459740000246, "nonpadding_tokens": 9919, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003474540000070192, "prefetch": {"data_wait_seconds": 0.005047370000170304, "consumed_batches": 505, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 510, "tokens": 46413533, "loss_tokens": 71241, "loss": 3.508429134908269, "lr": 0.0002999812592151665, "optimizer_lrs": [0.019998750614344432, 0.0002999812592151665], "gradient_norm": 0.5622676610946655, "seconds": 0.3888632900000175, "nonpadding_tokens_per_second": 183203.202338788, "ranks": [{"compute_seconds": 0.38503451700000824, "nonpadding_tokens": 12762, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.38495488600005956, "nonpadding_tokens": 3595, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.384647444000052, "nonpadding_tokens": 13605, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.38490184500005853, "nonpadding_tokens": 11882, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.38377784800002246, "nonpadding_tokens": 4353, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.38641853299998274, "nonpadding_tokens": 9244, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3845577609999964, "nonpadding_tokens": 3028, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3815410320000865, "nonpadding_tokens": 12772, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00037415200006307714, "prefetch": {"data_wait_seconds": 0.005057052000097428, "consumed_batches": 506, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 511, "tokens": 46467981, "loss_tokens": 54448, "loss": 3.5740623507750513, "lr": 0.00029998118187403764, "optimizer_lrs": [0.019998745458269178, 0.00029998118187403764], "gradient_norm": 0.4717079699039459, "seconds": 0.3605708649999997, "nonpadding_tokens_per_second": 151004.99037824382, "ranks": [{"compute_seconds": 0.27853716300000997, "nonpadding_tokens": 6434, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2818572219999851, "nonpadding_tokens": 8266, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2815639599999713, "nonpadding_tokens": 3757, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2823317809999253, "nonpadding_tokens": 5744, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.27944860000002336, "nonpadding_tokens": 10847, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2772416269999667, "nonpadding_tokens": 9371, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.28092059400000835, "nonpadding_tokens": 3499, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.27842484500001774, "nonpadding_tokens": 6530, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.05269357800000307, "prefetch": {"data_wait_seconds": 0.00506618800011438, "consumed_batches": 507, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 512, "tokens": 46530132, "loss_tokens": 62151, "loss": 3.3393795956621775, "lr": 0.0002999810933964786, "optimizer_lrs": [0.019998739559765243, 0.0002999810933964786], "gradient_norm": 0.44828519225120544, "seconds": 0.2556885010000087, "nonpadding_tokens_per_second": 243073.11340527542, "ranks": [{"compute_seconds": 0.24996867500010467, "nonpadding_tokens": 5915, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2524853009999788, "nonpadding_tokens": 6974, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.24904398800003946, "nonpadding_tokens": 5828, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2482368519999909, "nonpadding_tokens": 9455, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.24912754199999654, "nonpadding_tokens": 7208, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2532439499999555, "nonpadding_tokens": 4375, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.24981365400003597, "nonpadding_tokens": 9674, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.24947470600000088, "nonpadding_tokens": 12722, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036593200002243975, "prefetch": {"data_wait_seconds": 0.005076733000123568, "consumed_batches": 508, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 513, "tokens": 46596585, "loss_tokens": 66453, "loss": 3.1112670797405686, "lr": 0.0002999809985650969, "optimizer_lrs": [0.01999873323767313, 0.0002999809985650969], "gradient_norm": 0.6201344132423401, "seconds": 0.2959746479999694, "nonpadding_tokens_per_second": 224522.60843640522, "ranks": [{"compute_seconds": 0.29051618999994844, "nonpadding_tokens": 10645, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2872050070000114, "nonpadding_tokens": 1558, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.29014902799997344, "nonpadding_tokens": 13540, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2933062569999265, "nonpadding_tokens": 2078, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2901465069999176, "nonpadding_tokens": 8128, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2884279649999826, "nonpadding_tokens": 4962, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.290330076000032, "nonpadding_tokens": 14533, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2905858320000334, "nonpadding_tokens": 11009, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003489279999939754, "prefetch": {"data_wait_seconds": 0.005085994000182836, "consumed_batches": 509, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 514, "tokens": 46685982, "loss_tokens": 89397, "loss": 3.5299849267872525, "lr": 0.0002999808706174002, "optimizer_lrs": [0.01999872470782668, 0.0002999808706174002], "gradient_norm": 0.4690091013908386, "seconds": 0.3650255049999487, "nonpadding_tokens_per_second": 244906.17443296892, "ranks": [{"compute_seconds": 0.3614601839999523, "nonpadding_tokens": 10583, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3624568489999547, "nonpadding_tokens": 13802, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3615000710000231, "nonpadding_tokens": 8641, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36249970999995185, "nonpadding_tokens": 3350, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3623319380000112, "nonpadding_tokens": 2246, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.358751481000013, "nonpadding_tokens": 21359, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.36143751300005533, "nonpadding_tokens": 13525, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.361185450999983, "nonpadding_tokens": 15891, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004001759999709975, "prefetch": {"data_wait_seconds": 0.0050960500001338005, "consumed_batches": 510, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 515, "tokens": 46809185, "loss_tokens": 123203, "loss": 3.5131196074770905, "lr": 0.0002999806935820254, "optimizer_lrs": [0.019998712905468362, 0.0002999806935820254], "gradient_norm": 0.559686541557312, "seconds": 0.3785408250000728, "nonpadding_tokens_per_second": 325468.19751865946, "ranks": [{"compute_seconds": 0.3760074850000592, "nonpadding_tokens": 22466, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.37563627199995153, "nonpadding_tokens": 14346, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3750494579999213, "nonpadding_tokens": 15729, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3750662940000211, "nonpadding_tokens": 14302, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3729695069999934, "nonpadding_tokens": 11070, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3718367760000092, "nonpadding_tokens": 11998, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3740290939999795, "nonpadding_tokens": 14171, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3743971110000075, "nonpadding_tokens": 19121, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040597399993203, "prefetch": {"data_wait_seconds": 0.005105223000100523, "consumed_batches": 511, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 516, "tokens": 46905476, "loss_tokens": 96291, "loss": 3.40324251487678, "lr": 0.00029998055464992845, "optimizer_lrs": [0.019998703643328564, 0.00029998055464992845], "gradient_norm": 0.6066563129425049, "seconds": 0.4062436469999966, "nonpadding_tokens_per_second": 237027.7066757448, "ranks": [{"compute_seconds": 0.3678624069999614, "nonpadding_tokens": 24970, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3977784789999532, "nonpadding_tokens": 12150, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36782845300001554, "nonpadding_tokens": 4936, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3668906259999858, "nonpadding_tokens": 15711, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3699918039999375, "nonpadding_tokens": 4249, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.40303020499993636, "nonpadding_tokens": 8559, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.37032362800005103, "nonpadding_tokens": 17208, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37016289300004246, "nonpadding_tokens": 8508, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003827550000323754, "prefetch": {"data_wait_seconds": 0.005116723000014645, "consumed_batches": 512, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 517, "tokens": 47003242, "loss_tokens": 97766, "loss": 3.23291136489168, "lr": 0.0002999804130800839, "optimizer_lrs": [0.019998694205338927, 0.0002999804130800839], "gradient_norm": 0.46777525544166565, "seconds": 0.46424138200006837, "nonpadding_tokens_per_second": 210593.03153630884, "ranks": [{"compute_seconds": 0.4607648229999768, "nonpadding_tokens": 30294, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.4565046520000351, "nonpadding_tokens": 4579, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4579156869999679, "nonpadding_tokens": 13265, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.46053845999995247, "nonpadding_tokens": 8861, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4599311069999885, "nonpadding_tokens": 20174, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.46141757199995936, "nonpadding_tokens": 4985, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4602851219999593, "nonpadding_tokens": 10585, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.45720029599999634, "nonpadding_tokens": 5023, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003645010000354887, "prefetch": {"data_wait_seconds": 0.00512653299995236, "consumed_batches": 513, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 518, "tokens": 47115867, "loss_tokens": 112625, "loss": 3.161337125416204, "lr": 0.0002999802493572768, "optimizer_lrs": [0.019998683290485122, 0.0002999802493572768], "gradient_norm": 0.46217668056488037, "seconds": 0.40626080100003037, "nonpadding_tokens_per_second": 277223.39867092314, "ranks": [{"compute_seconds": 0.4017272259999345, "nonpadding_tokens": 21666, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3988840499999924, "nonpadding_tokens": 21070, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.40066393999995853, "nonpadding_tokens": 11727, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4039670350000506, "nonpadding_tokens": 6889, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.40240001699999084, "nonpadding_tokens": 10442, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3999938839999686, "nonpadding_tokens": 10272, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4032036500000231, "nonpadding_tokens": 13761, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3994842630000903, "nonpadding_tokens": 16798, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003562489999922036, "prefetch": {"data_wait_seconds": 0.00513563099991643, "consumed_batches": 514, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 519, "tokens": 47191068, "loss_tokens": 75201, "loss": 3.9586470758367573, "lr": 0.0002999801396583188, "optimizer_lrs": [0.019998675977221256, 0.0002999801396583188], "gradient_norm": 0.5120313167572021, "seconds": 0.29747242299993104, "nonpadding_tokens_per_second": 252799.9040772174, "ranks": [{"compute_seconds": 0.29494663399998444, "nonpadding_tokens": 4948, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2950780930000292, "nonpadding_tokens": 8218, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.29372407799996836, "nonpadding_tokens": 3914, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2950640740000381, "nonpadding_tokens": 11960, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.29375113700007205, "nonpadding_tokens": 12236, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2912380249999842, "nonpadding_tokens": 6271, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2926621529999238, "nonpadding_tokens": 8987, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2921166109999831, "nonpadding_tokens": 18667, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039487200001531164, "prefetch": {"data_wait_seconds": 0.005146082999885948, "consumed_batches": 515, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 520, "tokens": 47266913, "loss_tokens": 75845, "loss": 3.374627735842837, "lr": 0.0002999800287122375, "optimizer_lrs": [0.019998668580815834, 0.0002999800287122375], "gradient_norm": 1.2130045890808105, "seconds": 0.36042483399990033, "nonpadding_tokens_per_second": 210432.22565518605, "ranks": [{"compute_seconds": 0.3542770140001039, "nonpadding_tokens": 2142, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3572776779999458, "nonpadding_tokens": 17202, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3535954789999778, "nonpadding_tokens": 3287, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3519550439999648, "nonpadding_tokens": 7966, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3557796119999921, "nonpadding_tokens": 19039, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3579791219999606, "nonpadding_tokens": 8196, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3564662119999866, "nonpadding_tokens": 12374, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3542287259999739, "nonpadding_tokens": 5639, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003682970000227215, "prefetch": {"data_wait_seconds": 0.005155523999974321, "consumed_batches": 516, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 521, "tokens": 47421599, "loss_tokens": 154686, "loss": 3.122066638221946, "lr": 0.00029997980147975634, "optimizer_lrs": [0.019998653431983756, 0.00029997980147975634], "gradient_norm": 0.8676220178604126, "seconds": 0.3187833719999844, "nonpadding_tokens_per_second": 485238.6089949747, "ranks": [{"compute_seconds": 0.3138477879999755, "nonpadding_tokens": 16927, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3123648639999601, "nonpadding_tokens": 7883, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3162068450000106, "nonpadding_tokens": 17921, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31637527099996987, "nonpadding_tokens": 23989, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3149175759999707, "nonpadding_tokens": 19993, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3122560180000846, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3152456660000098, "nonpadding_tokens": 16086, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3151894249999714, "nonpadding_tokens": 19119, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003901540000015302, "prefetch": {"data_wait_seconds": 0.005164757999978065, "consumed_batches": 517, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 522, "tokens": 47521769, "loss_tokens": 100170, "loss": 3.804993074273735, "lr": 0.00029997965364517187, "optimizer_lrs": [0.01999864357634479, 0.00029997965364517187], "gradient_norm": 0.6995624899864197, "seconds": 0.809381816000041, "nonpadding_tokens_per_second": 123761.11993105974, "ranks": [{"compute_seconds": 0.8063774290000083, "nonpadding_tokens": 15239, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.8021219249999376, "nonpadding_tokens": 3703, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.8051009980000572, "nonpadding_tokens": 13372, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.807014120999952, "nonpadding_tokens": 19391, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.8030027319999817, "nonpadding_tokens": 11661, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.8036226369999895, "nonpadding_tokens": 8614, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.8068479880000723, "nonpadding_tokens": 15999, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.8033124520000001, "nonpadding_tokens": 12191, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038934200006224273, "prefetch": {"data_wait_seconds": 0.0051745669999263555, "consumed_batches": 518, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 523, "tokens": 47596377, "loss_tokens": 74608, "loss": 3.0950354687164916, "lr": 0.0002999795431857023, "optimizer_lrs": [0.019998636212380152, 0.0002999795431857023], "gradient_norm": 0.46769753098487854, "seconds": 0.28036290599993663, "nonpadding_tokens_per_second": 266112.2366880334, "ranks": [{"compute_seconds": 0.2746403140000666, "nonpadding_tokens": 6917, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2779139619999569, "nonpadding_tokens": 6945, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2738822669999763, "nonpadding_tokens": 8605, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2767873589999681, "nonpadding_tokens": 11482, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.27692301199999747, "nonpadding_tokens": 10979, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.27325691600003665, "nonpadding_tokens": 15315, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.27645895500006645, "nonpadding_tokens": 11054, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.276978455999938, "nonpadding_tokens": 3311, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00040866899996672146, "prefetch": {"data_wait_seconds": 0.005185559999972611, "consumed_batches": 519, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 524, "tokens": 47699007, "loss_tokens": 102630, "loss": 3.1782507551398225, "lr": 0.0002999793907502369, "optimizer_lrs": [0.019998626050015792, 0.0002999793907502369], "gradient_norm": 0.45578068494796753, "seconds": 0.29764218599996184, "nonpadding_tokens_per_second": 344809.9927609494, "ranks": [{"compute_seconds": 0.29487292199996773, "nonpadding_tokens": 15916, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2949990740000885, "nonpadding_tokens": 15203, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2935531069999797, "nonpadding_tokens": 8544, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2949242409999897, "nonpadding_tokens": 13437, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2937861319999229, "nonpadding_tokens": 25118, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2914790780000658, "nonpadding_tokens": 3577, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.29462042300008306, "nonpadding_tokens": 2677, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2937830869999516, "nonpadding_tokens": 18158, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0005176189999929193, "prefetch": {"data_wait_seconds": 0.0051945230000001175, "consumed_batches": 520, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 525, "tokens": 47782922, "loss_tokens": 83915, "loss": 3.1473146189596615, "lr": 0.00029997926569156393, "optimizer_lrs": [0.01999861771277093, 0.00029997926569156393], "gradient_norm": 0.46386516094207764, "seconds": 0.2666487960000268, "nonpadding_tokens_per_second": 314702.3397772686, "ranks": [{"compute_seconds": 0.25998893899998166, "nonpadding_tokens": 5481, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2633170929999551, "nonpadding_tokens": 10937, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.25978935399996317, "nonpadding_tokens": 7256, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2583598029999621, "nonpadding_tokens": 17955, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2622425000000703, "nonpadding_tokens": 3839, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.26416231700000026, "nonpadding_tokens": 10330, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2620430640000677, "nonpadding_tokens": 14504, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.26218256499998915, "nonpadding_tokens": 13613, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039555599994400836, "prefetch": {"data_wait_seconds": 0.0052036120000593655, "consumed_batches": 521, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 526, "tokens": 47899751, "loss_tokens": 116829, "loss": 3.8214666307166887, "lr": 0.0002999790909511989, "optimizer_lrs": [0.019998606063413263, 0.0002999790909511989], "gradient_norm": 0.6582545638084412, "seconds": 0.387029779000045, "nonpadding_tokens_per_second": 301860.49327224103, "ranks": [{"compute_seconds": 0.3597786500000666, "nonpadding_tokens": 20958, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.35673003400006564, "nonpadding_tokens": 10189, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.357364232000009, "nonpadding_tokens": 17083, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36004784599992945, "nonpadding_tokens": 10227, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3590148929999941, "nonpadding_tokens": 13221, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.35666331700008413, "nonpadding_tokens": 12142, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.35873537400004807, "nonpadding_tokens": 19843, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3600474680000616, "nonpadding_tokens": 13166, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.02489990900005523, "prefetch": {"data_wait_seconds": 0.0052131490000419944, "consumed_batches": 522, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 527, "tokens": 47989934, "loss_tokens": 90183, "loss": 3.630627099896876, "lr": 0.0002999789555636649, "optimizer_lrs": [0.01999859703757766, 0.0002999789555636649], "gradient_norm": 0.4468296468257904, "seconds": 0.3671585770000547, "nonpadding_tokens_per_second": 245624.11352843477, "ranks": [{"compute_seconds": 0.36212054400004945, "nonpadding_tokens": 11360, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3647362689999909, "nonpadding_tokens": 20120, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3607573949999505, "nonpadding_tokens": 2049, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36472669699992366, "nonpadding_tokens": 11012, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36070527099991523, "nonpadding_tokens": 17772, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3607703399999309, "nonpadding_tokens": 11731, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3636342679999416, "nonpadding_tokens": 4677, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.36340126499999315, "nonpadding_tokens": 11462, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038647600001695537, "prefetch": {"data_wait_seconds": 0.005221967000011318, "consumed_batches": 523, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 528, "tokens": 48047874, "loss_tokens": 57940, "loss": 3.3453081851915774, "lr": 0.0002999788683505398, "optimizer_lrs": [0.019998591223369323, 0.0002999788683505398], "gradient_norm": 0.4713922142982483, "seconds": 0.270008293999922, "nonpadding_tokens_per_second": 214586.0008286143, "ranks": [{"compute_seconds": 0.26361891399994875, "nonpadding_tokens": 4015, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2673328920000131, "nonpadding_tokens": 2559, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2639466309999534, "nonpadding_tokens": 19702, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2676443059999656, "nonpadding_tokens": 1782, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2660736310000402, "nonpadding_tokens": 8125, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.2633114970000179, "nonpadding_tokens": 5402, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2662788119999959, "nonpadding_tokens": 8625, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.26397007099990333, "nonpadding_tokens": 7730, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039642999990974204, "prefetch": {"data_wait_seconds": 0.005233406000002105, "consumed_batches": 524, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 529, "tokens": 48120205, "loss_tokens": 72331, "loss": 3.2546798744659964, "lr": 0.0002999787592225505, "optimizer_lrs": [0.019998583948170036, 0.0002999787592225505], "gradient_norm": 0.45674654841423035, "seconds": 0.42124928499993075, "nonpadding_tokens_per_second": 171705.92942374226, "ranks": [{"compute_seconds": 0.41766681400008565, "nonpadding_tokens": 17668, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.41436218100000133, "nonpadding_tokens": 5661, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4143932719999839, "nonpadding_tokens": 12781, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.41886757000008856, "nonpadding_tokens": 6306, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.417267815999935, "nonpadding_tokens": 6386, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.41498214099999586, "nonpadding_tokens": 9499, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.4181068679999953, "nonpadding_tokens": 10102, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.4169736680000824, "nonpadding_tokens": 3928, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038822400006210955, "prefetch": {"data_wait_seconds": 0.005242535999968823, "consumed_batches": 525, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 530, "tokens": 48195780, "loss_tokens": 75575, "loss": 3.367196493549454, "lr": 0.0002999786449000187, "optimizer_lrs": [0.019998576326667915, 0.0002999786449000187], "gradient_norm": 0.47978225350379944, "seconds": 0.37812562200008415, "nonpadding_tokens_per_second": 199867.43982131733, "ranks": [{"compute_seconds": 0.3717707270000119, "nonpadding_tokens": 19943, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3757887579999988, "nonpadding_tokens": 3067, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3704316260000269, "nonpadding_tokens": 14211, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3702305229999183, "nonpadding_tokens": 8987, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3725709899999856, "nonpadding_tokens": 7696, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3711086600000044, "nonpadding_tokens": 1077, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.37401900700001534, "nonpadding_tokens": 15418, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37381810800002313, "nonpadding_tokens": 5176, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003530909999653886, "prefetch": {"data_wait_seconds": 0.005250441999919531, "consumed_batches": 526, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 531, "tokens": 48319095, "loss_tokens": 123315, "loss": 3.6608946093338197, "lr": 0.0002999784577023179, "optimizer_lrs": [0.019998563846821193, 0.0002999784577023179], "gradient_norm": 0.45409998297691345, "seconds": 0.33035564700003306, "nonpadding_tokens_per_second": 373279.5280475035, "ranks": [{"compute_seconds": 0.32441136400007053, "nonpadding_tokens": 17409, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.32733839499996975, "nonpadding_tokens": 14098, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3238306090000833, "nonpadding_tokens": 18775, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3278112320000446, "nonpadding_tokens": 22290, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.32580119400006424, "nonpadding_tokens": 6224, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.32788818000005904, "nonpadding_tokens": 6390, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.32636019500000657, "nonpadding_tokens": 12261, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3259410759999355, "nonpadding_tokens": 25868, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00036508699997739313, "prefetch": {"data_wait_seconds": 0.00526115699994989, "consumed_batches": 527, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 532, "tokens": 48391790, "loss_tokens": 72695, "loss": 3.753068041130752, "lr": 0.0002999783469653476, "optimizer_lrs": [0.019998556464356505, 0.0002999783469653476], "gradient_norm": 0.48527443408966064, "seconds": 0.3801825320000489, "nonpadding_tokens_per_second": 191210.78398202325, "ranks": [{"compute_seconds": 0.3751571390000663, "nonpadding_tokens": 8838, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.37784638699997686, "nonpadding_tokens": 10313, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.37401433200000156, "nonpadding_tokens": 6311, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3721662110000352, "nonpadding_tokens": 13312, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37572412800000166, "nonpadding_tokens": 17915, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37727373500001704, "nonpadding_tokens": 6795, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3731180760000825, "nonpadding_tokens": 4481, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3759670869999354, "nonpadding_tokens": 4730, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038013300002148753, "prefetch": {"data_wait_seconds": 0.005270129999985329, "consumed_batches": 528, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 533, "tokens": 48475482, "loss_tokens": 83692, "loss": 3.220834652654973, "lr": 0.0002999782191250168, "optimizer_lrs": [0.019998547941667787, 0.0002999782191250168], "gradient_norm": 0.5056270360946655, "seconds": 0.3495127639999964, "nonpadding_tokens_per_second": 239453.3436838972, "ranks": [{"compute_seconds": 0.3414469960000588, "nonpadding_tokens": 13009, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34452524699997866, "nonpadding_tokens": 10520, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.34112013100002514, "nonpadding_tokens": 13840, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.34424742300006983, "nonpadding_tokens": 12262, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.34155295900006877, "nonpadding_tokens": 9909, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3405402379999032, "nonpadding_tokens": 7338, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.34126870000000054, "nonpadding_tokens": 5293, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.34686443399994005, "nonpadding_tokens": 11521, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003765829999338166, "prefetch": {"data_wait_seconds": 0.005280719999973371, "consumed_batches": 529, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 534, "tokens": 48579123, "loss_tokens": 103641, "loss": 3.469455259019114, "lr": 0.0002999780602909382, "optimizer_lrs": [0.019998537352729214, 0.0002999780602909382], "gradient_norm": 0.6921099424362183, "seconds": 0.46883288499998343, "nonpadding_tokens_per_second": 221061.71157341846, "ranks": [{"compute_seconds": 0.4506807299999309, "nonpadding_tokens": 18695, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.45158391199993275, "nonpadding_tokens": 19629, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.4489672029999383, "nonpadding_tokens": 9249, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.4516255959999853, "nonpadding_tokens": 14604, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.4474536249999801, "nonpadding_tokens": 3925, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.4518759399999226, "nonpadding_tokens": 8724, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.45021066399999654, "nonpadding_tokens": 17368, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.45022914899993793, "nonpadding_tokens": 11447, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.012896349000016016, "prefetch": {"data_wait_seconds": 0.005290731000059168, "consumed_batches": 530, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 535, "tokens": 48695996, "loss_tokens": 116873, "loss": 3.6710481141923283, "lr": 0.0002999778804860887, "optimizer_lrs": [0.019998525365739248, 0.0002999778804860887], "gradient_norm": 0.47453874349594116, "seconds": 0.3838742650000313, "nonpadding_tokens_per_second": 304456.460502739, "ranks": [{"compute_seconds": 0.3814405530000613, "nonpadding_tokens": 25603, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3814467910000303, "nonpadding_tokens": 14757, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3778647159999764, "nonpadding_tokens": 26134, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.381444672999919, "nonpadding_tokens": 12518, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37739375800003927, "nonpadding_tokens": 12429, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37758988200005206, "nonpadding_tokens": 6549, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.37763290199995936, "nonpadding_tokens": 4641, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.38027537999994365, "nonpadding_tokens": 14242, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0004018890000452302, "prefetch": {"data_wait_seconds": 0.005300097000031201, "consumed_batches": 531, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 536, "tokens": 48788402, "loss_tokens": 92406, "loss": 3.7090932271713957, "lr": 0.0002999777378034295, "optimizer_lrs": [0.01999851585356197, 0.0002999777378034295], "gradient_norm": 0.6340365409851074, "seconds": 0.25267118700003266, "nonpadding_tokens_per_second": 365716.41229511483, "ranks": [{"compute_seconds": 0.20018520600001466, "nonpadding_tokens": 5845, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.22253528399994593, "nonpadding_tokens": 9245, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2220967750000682, "nonpadding_tokens": 3965, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.22641134399998464, "nonpadding_tokens": 3317, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2244213569999829, "nonpadding_tokens": 15767, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.22784148799996728, "nonpadding_tokens": 22400, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.22554263199992874, "nonpadding_tokens": 18785, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.22445984199998747, "nonpadding_tokens": 13082, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0017213959999935469, "prefetch": {"data_wait_seconds": 0.005309328000066671, "consumed_batches": 532, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 537, "tokens": 48891658, "loss_tokens": 103256, "loss": 2.810846343069652, "lr": 0.0002999775778248361, "optimizer_lrs": [0.019998505188322406, 0.0002999775778248361], "gradient_norm": 0.4645576775074005, "seconds": 0.36848500200005674, "nonpadding_tokens_per_second": 280217.64641586173, "ranks": [{"compute_seconds": 0.36536667600000783, "nonpadding_tokens": 26161, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.362096160999954, "nonpadding_tokens": 10226, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.36228872699996373, "nonpadding_tokens": 13888, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.36617354999998497, "nonpadding_tokens": 11963, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.36562755400007063, "nonpadding_tokens": 12169, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.36278502000004664, "nonpadding_tokens": 16945, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3661393190000126, "nonpadding_tokens": 5829, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3650469089999433, "nonpadding_tokens": 6075, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003809710000268751, "prefetch": {"data_wait_seconds": 0.005319235000115441, "consumed_batches": 533, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 538, "tokens": 49007995, "loss_tokens": 116337, "loss": 3.1979607197194357, "lr": 0.0002999773968931993, "optimizer_lrs": [0.019998493126213285, 0.0002999773968931993], "gradient_norm": 0.4689185917377472, "seconds": 0.3087197950000018, "nonpadding_tokens_per_second": 376836.8659353357, "ranks": [{"compute_seconds": 0.30642218399998455, "nonpadding_tokens": 7124, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3064701609999929, "nonpadding_tokens": 17683, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.30555999700004577, "nonpadding_tokens": 21277, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.30645652000009704, "nonpadding_tokens": 19156, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30537782199996855, "nonpadding_tokens": 16591, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.302913129999979, "nonpadding_tokens": 10313, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.30585626200002025, "nonpadding_tokens": 5006, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3036500399999795, "nonpadding_tokens": 19187, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00035499500006608287, "prefetch": {"data_wait_seconds": 0.0053283710001323925, "consumed_batches": 534, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 539, "tokens": 49114168, "loss_tokens": 106173, "loss": 3.5642898265095644, "lr": 0.0002999772311344879, "optimizer_lrs": [0.01999848207563253, 0.0002999772311344879], "gradient_norm": 0.5163856148719788, "seconds": 0.342515280999919, "nonpadding_tokens_per_second": 309980.33048348903, "ranks": [{"compute_seconds": 0.33889921499996944, "nonpadding_tokens": 11235, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.34023615099999915, "nonpadding_tokens": 16446, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33668809200003125, "nonpadding_tokens": 13020, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.33574239500001113, "nonpadding_tokens": 29734, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33868459300003906, "nonpadding_tokens": 4827, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3362938000000213, "nonpadding_tokens": 9937, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3391596019999952, "nonpadding_tokens": 3451, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.33944681399998444, "nonpadding_tokens": 17523, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003668089999564472, "prefetch": {"data_wait_seconds": 0.0053399160001390555, "consumed_batches": 535, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 540, "tokens": 49192337, "loss_tokens": 78169, "loss": 3.554709187785439, "lr": 0.0002999771087089795, "optimizer_lrs": [0.01999847391393197, 0.0002999771087089795], "gradient_norm": 0.5542816519737244, "seconds": 0.34968238599992674, "nonpadding_tokens_per_second": 223542.85811815635, "ranks": [{"compute_seconds": 0.2569809689999829, "nonpadding_tokens": 11283, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.2656836689999409, "nonpadding_tokens": 11022, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.2624819910000724, "nonpadding_tokens": 11001, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.2681053649999967, "nonpadding_tokens": 14982, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.2628925529999151, "nonpadding_tokens": 16935, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.26184784400004446, "nonpadding_tokens": 5540, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.2649805789999391, "nonpadding_tokens": 5475, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.2630836050000198, "nonpadding_tokens": 1931, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.05030894799995167, "prefetch": {"data_wait_seconds": 0.005350229000100626, "consumed_batches": 536, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 541, "tokens": 49264576, "loss_tokens": 72239, "loss": 3.4072164101108817, "lr": 0.0002999769952790071, "optimizer_lrs": [0.019998466351933808, 0.0002999769952790071], "gradient_norm": 0.7074565291404724, "seconds": 0.5110620389999667, "nonpadding_tokens_per_second": 141350.74509027408, "ranks": [{"compute_seconds": 0.5063465709999946, "nonpadding_tokens": 14870, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.5086645889999772, "nonpadding_tokens": 9012, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.5052576489999865, "nonpadding_tokens": 1961, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.5087045340000032, "nonpadding_tokens": 13726, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.5072173230000772, "nonpadding_tokens": 8533, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.5052939360000437, "nonpadding_tokens": 6711, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.5075113149999879, "nonpadding_tokens": 7063, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.5074650010000141, "nonpadding_tokens": 10363, "peak_allocated_bytes": 14987615744, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00039006800000152, "prefetch": {"data_wait_seconds": 0.0053593170000567625, "consumed_batches": 537, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 542, "tokens": 49331266, "loss_tokens": 66690, "loss": 3.4035008059679113, "lr": 0.00029997689031325463, "optimizer_lrs": [0.019998459354216975, 0.00029997689031325463], "gradient_norm": 1.2139307260513306, "seconds": 0.3215845119999585, "nonpadding_tokens_per_second": 207379.39021145584, "ranks": [{"compute_seconds": 0.30875729399997454, "nonpadding_tokens": 10960, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3191046690000121, "nonpadding_tokens": 10904, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3146161290000009, "nonpadding_tokens": 4085, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3192998810000063, "nonpadding_tokens": 3299, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.31515703799993844, "nonpadding_tokens": 6587, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31519138700002713, "nonpadding_tokens": 7829, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3157767349999858, "nonpadding_tokens": 10541, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.31806003099995905, "nonpadding_tokens": 12485, "peak_allocated_bytes": 14988160512, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003488190000098257, "prefetch": {"data_wait_seconds": 0.005368177000150354, "consumed_batches": 538, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 543, "tokens": 49418721, "loss_tokens": 87455, "loss": 3.3294748013263966, "lr": 0.00029997675230267006, "optimizer_lrs": [0.01999845015351134, 0.00029997675230267006], "gradient_norm": 0.5792829990386963, "seconds": 0.31641336699999556, "nonpadding_tokens_per_second": 276394.7706419155, "ranks": [{"compute_seconds": 0.31158507800000734, "nonpadding_tokens": 3349, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3131289939999533, "nonpadding_tokens": 6979, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.309945899000013, "nonpadding_tokens": 2443, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.31405283200001577, "nonpadding_tokens": 9193, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3128832049999346, "nonpadding_tokens": 16383, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.31399069600001894, "nonpadding_tokens": 14260, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.31252820300005624, "nonpadding_tokens": 15852, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.30907986199997595, "nonpadding_tokens": 18996, "peak_allocated_bytes": 14988160512, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003603220000059082, "prefetch": {"data_wait_seconds": 0.005377428000201689, "consumed_batches": 539, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 544, "tokens": 49502978, "loss_tokens": 84257, "loss": 3.249660636504979, "lr": 0.00029997661895019397, "optimizer_lrs": [0.019998441263346267, 0.00029997661895019397], "gradient_norm": 1.6109180450439453, "seconds": 0.3777365590000272, "nonpadding_tokens_per_second": 223057.57277784153, "ranks": [{"compute_seconds": 0.371732049000002, "nonpadding_tokens": 1251, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3751212789999272, "nonpadding_tokens": 13005, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3705452020000166, "nonpadding_tokens": 12072, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37406077399998594, "nonpadding_tokens": 29411, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37122713399992335, "nonpadding_tokens": 4839, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3701504399999749, "nonpadding_tokens": 4726, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3715418659999159, "nonpadding_tokens": 10103, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37162329399995997, "nonpadding_tokens": 8850, "peak_allocated_bytes": 14988160512, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0006074520000538541, "prefetch": {"data_wait_seconds": 0.005390860000261455, "consumed_batches": 540, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 545, "tokens": 49634813, "loss_tokens": 131835, "loss": 3.325173986422422, "lr": 0.0002999764095314894, "optimizer_lrs": [0.019998427302099296, 0.0002999764095314894], "gradient_norm": 0.5186595916748047, "seconds": 0.3095717720000266, "nonpadding_tokens_per_second": 425862.47172429104, "ranks": [{"compute_seconds": 0.30688685600000554, "nonpadding_tokens": 8874, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3072785400000839, "nonpadding_tokens": 14546, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3067898790000072, "nonpadding_tokens": 3575, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3072753939999302, "nonpadding_tokens": 24697, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.30690692499990746, "nonpadding_tokens": 24061, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3034886299999471, "nonpadding_tokens": 18398, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.30412004400000114, "nonpadding_tokens": 16658, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3048289380000142, "nonpadding_tokens": 21026, "peak_allocated_bytes": 14988160512, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038113999994493497, "prefetch": {"data_wait_seconds": 0.005411226000319402, "consumed_batches": 541, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 546, "tokens": 49749208, "loss_tokens": 114395, "loss": 3.329248710607981, "lr": 0.000299976227059553, "optimizer_lrs": [0.019998415137303535, 0.000299976227059553], "gradient_norm": 0.47315946221351624, "seconds": 0.40081301100008204, "nonpadding_tokens_per_second": 285407.40160747076, "ranks": [{"compute_seconds": 0.3981658239999888, "nonpadding_tokens": 19168, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3979199030000018, "nonpadding_tokens": 21739, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.394303243999957, "nonpadding_tokens": 11460, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3984175429999368, "nonpadding_tokens": 2855, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.39684788700003537, "nonpadding_tokens": 10869, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3943029670000442, "nonpadding_tokens": 10836, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3946462479999582, "nonpadding_tokens": 19549, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3938821500000813, "nonpadding_tokens": 17919, "peak_allocated_bytes": 14988160512, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003753050000341318, "prefetch": {"data_wait_seconds": 0.005419718000212015, "consumed_batches": 542, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 547, "tokens": 49860290, "loss_tokens": 111082, "loss": 3.313359725248015, "lr": 0.00029997604919950767, "optimizer_lrs": [0.01999840327996718, 0.00029997604919950767], "gradient_norm": 0.43487295508384705, "seconds": 0.3754855320000843, "nonpadding_tokens_per_second": 295835.6328892456, "ranks": [{"compute_seconds": 0.372754212000018, "nonpadding_tokens": 15356, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.36949911699991844, "nonpadding_tokens": 925, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.3726962509999794, "nonpadding_tokens": 17727, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3731685569999854, "nonpadding_tokens": 9606, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.37171154999998635, "nonpadding_tokens": 27433, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.37006391499994606, "nonpadding_tokens": 9521, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.3727244529999325, "nonpadding_tokens": 15783, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37297335799996745, "nonpadding_tokens": 14731, "peak_allocated_bytes": 14988160512, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.00038710799992713874, "prefetch": {"data_wait_seconds": 0.0054297680002264315, "consumed_batches": 543, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 548, "tokens": 49977182, "loss_tokens": 116892, "loss": 3.500580931543647, "lr": 0.00029997586132101187, "optimizer_lrs": [0.019998390754734127, 0.00029997586132101187], "gradient_norm": 0.4376978576183319, "seconds": 0.3801711820000264, "nonpadding_tokens_per_second": 307472.01664536446, "ranks": [{"compute_seconds": 0.37571795899998506, "nonpadding_tokens": 21121, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.3724765129999241, "nonpadding_tokens": 10679, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.37419927700000244, "nonpadding_tokens": 18507, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.37772497399998883, "nonpadding_tokens": 12218, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.3764520369999218, "nonpadding_tokens": 17554, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.3760540079999828, "nonpadding_tokens": 5707, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.37663449900003343, "nonpadding_tokens": 14488, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.37683907400003136, "nonpadding_tokens": 16618, "peak_allocated_bytes": 14988160512, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003529049999997369, "prefetch": {"data_wait_seconds": 0.0054412600002251565, "consumed_batches": 544, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 549, "tokens": 50065393, "loss_tokens": 88211, "loss": 2.9961115960594484, "lr": 0.00029997571905507597, "optimizer_lrs": [0.0199983812703384, 0.00029997571905507597], "gradient_norm": 0.8112643957138062, "seconds": 0.33968342299999676, "nonpadding_tokens_per_second": 259685.91349246042, "ranks": [{"compute_seconds": 0.32888021999997363, "nonpadding_tokens": 14263, "peak_allocated_bytes": 15006568960, "peak_reserved_bytes": 18169724928}, {"compute_seconds": 0.33660470800009534, "nonpadding_tokens": 2962, "peak_allocated_bytes": 14994473472, "peak_reserved_bytes": 19725811712}, {"compute_seconds": 0.33304276299998037, "nonpadding_tokens": 7714, "peak_allocated_bytes": 14998106624, "peak_reserved_bytes": 20201865216}, {"compute_seconds": 0.3314280330000656, "nonpadding_tokens": 14076, "peak_allocated_bytes": 15000323584, "peak_reserved_bytes": 20860370944}, {"compute_seconds": 0.33550502299999607, "nonpadding_tokens": 6823, "peak_allocated_bytes": 14986161664, "peak_reserved_bytes": 19235078144}, {"compute_seconds": 0.336586681999961, "nonpadding_tokens": 25188, "peak_allocated_bytes": 14989873664, "peak_reserved_bytes": 19197329408}, {"compute_seconds": 0.33345962300006704, "nonpadding_tokens": 12866, "peak_allocated_bytes": 14994550272, "peak_reserved_bytes": 19662897152}, {"compute_seconds": 0.3346630839999989, "nonpadding_tokens": 4319, "peak_allocated_bytes": 14988160512, "peak_reserved_bytes": 21260926976}], "data_fetch_seconds": 0.0003728280000814266, "prefetch": {"data_wait_seconds": 0.005450986000141711, "consumed_batches": 545, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 550, "tokens": 50154530, "loss_tokens": 89137, "loss": 3.0615120544779386, "lr": 0.0002999755748711236, "optimizer_lrs": [0.019998371658074908, 0.0002999755748711236], "gradient_norm": 0.5230324268341064, "seconds": 129.575096675, "nonpadding_tokens_per_second": 687.9176808455196, "ranks": [{"compute_seconds": 128.300779859, "nonpadding_tokens": 25550, "peak_allocated_bytes": 14992181760, "peak_reserved_bytes": 18232639488}, {"compute_seconds": 128.26288513500003, "nonpadding_tokens": 16720, "peak_allocated_bytes": 14433403392, "peak_reserved_bytes": 16326328320}, {"compute_seconds": 128.262562763, "nonpadding_tokens": 6533, "peak_allocated_bytes": 9982914560, "peak_reserved_bytes": 10930356224}, {"compute_seconds": 128.262801237, "nonpadding_tokens": 7760, "peak_allocated_bytes": 10440600064, "peak_reserved_bytes": 11337203712}, {"compute_seconds": 128.262614804, "nonpadding_tokens": 17354, "peak_allocated_bytes": 14433403392, "peak_reserved_bytes": 16326328320}, {"compute_seconds": 128.26210941200003, "nonpadding_tokens": 2113, "peak_allocated_bytes": 4939965952, "peak_reserved_bytes": 5433720832}, {"compute_seconds": 128.26272032699998, "nonpadding_tokens": 8106, "peak_allocated_bytes": 8850921984, "peak_reserved_bytes": 9529458688}, {"compute_seconds": 128.262650839, "nonpadding_tokens": 5001, "peak_allocated_bytes": 5246515200, "peak_reserved_bytes": 5786042368}], "data_fetch_seconds": 0.0435756170000019, "prefetch": {"data_wait_seconds": 1.7916000004447596e-05, "consumed_batches": 1, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 551, "tokens": 50249247, "loss_tokens": 94717, "loss": 3.4592364623034935, "lr": 0.0002999754211935174, "optimizer_lrs": [0.019998361412901164, 0.0002999754211935174], "gradient_norm": 0.5908527374267578, "seconds": 49.12730026900002, "nonpadding_tokens_per_second": 1927.9911471090484, "ranks": [{"compute_seconds": 49.12291606900001, "nonpadding_tokens": 10506, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18234736640}, {"compute_seconds": 49.12464207099998, "nonpadding_tokens": 8533, "peak_allocated_bytes": 14433403392, "peak_reserved_bytes": 16326328320}, {"compute_seconds": 49.12177232500002, "nonpadding_tokens": 6902, "peak_allocated_bytes": 9982914560, "peak_reserved_bytes": 10930356224}, {"compute_seconds": 49.12422474600001, "nonpadding_tokens": 15965, "peak_allocated_bytes": 14980128256, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 49.121422205000016, "nonpadding_tokens": 9732, "peak_allocated_bytes": 14466909696, "peak_reserved_bytes": 16326328320}, {"compute_seconds": 49.12196426899999, "nonpadding_tokens": 20617, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 49.12400566400001, "nonpadding_tokens": 10057, "peak_allocated_bytes": 10150817792, "peak_reserved_bytes": 12171870208}, {"compute_seconds": 49.12125484799998, "nonpadding_tokens": 12405, "peak_allocated_bytes": 14466570752, "peak_reserved_bytes": 15644753920}], "data_fetch_seconds": 0.000513865000016267, "prefetch": {"data_wait_seconds": 3.5401000005208516e-05, "consumed_batches": 2, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 552, "tokens": 50329492, "loss_tokens": 80245, "loss": 3.3253637298274037, "lr": 0.0002999752906195454, "optimizer_lrs": [0.019998352707969697, 0.0002999752906195454], "gradient_norm": 0.4518713653087616, "seconds": 0.3972156959999893, "nonpadding_tokens_per_second": 202018.70371205613, "ranks": [{"compute_seconds": 0.3784241550000047, "nonpadding_tokens": 17902, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18234736640}, {"compute_seconds": 0.3837640020000208, "nonpadding_tokens": 4790, "peak_allocated_bytes": 14433403392, "peak_reserved_bytes": 16326328320}, {"compute_seconds": 0.3810540839999703, "nonpadding_tokens": 14907, "peak_allocated_bytes": 14983958016, "peak_reserved_bytes": 18280873984}, {"compute_seconds": 0.3839258839999502, "nonpadding_tokens": 2518, "peak_allocated_bytes": 14980128256, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.3847217329999921, "nonpadding_tokens": 5637, "peak_allocated_bytes": 14466909696, "peak_reserved_bytes": 16366174208}, {"compute_seconds": 0.38225558900001033, "nonpadding_tokens": 12373, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3803269140000225, "nonpadding_tokens": 10028, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17553162240}, {"compute_seconds": 0.3849865720000025, "nonpadding_tokens": 12090, "peak_allocated_bytes": 14466570752, "peak_reserved_bytes": 17792237568}], "data_fetch_seconds": 0.0046336849999875085, "prefetch": {"data_wait_seconds": 4.535600000110662e-05, "consumed_batches": 3, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 553, "tokens": 50412022, "loss_tokens": 82530, "loss": 3.2879346904156064, "lr": 0.00029997515596664256, "optimizer_lrs": [0.019998343731109505, 0.00029997515596664256], "gradient_norm": 0.44000735878944397, "seconds": 0.34075278300002765, "nonpadding_tokens_per_second": 242199.0490390017, "ranks": [{"compute_seconds": 0.3381162550000454, "nonpadding_tokens": 4523, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18234736640}, {"compute_seconds": 0.3377619010000217, "nonpadding_tokens": 9480, "peak_allocated_bytes": 14466909696, "peak_reserved_bytes": 16326328320}, {"compute_seconds": 0.3349483410000289, "nonpadding_tokens": 18207, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18280873984}, {"compute_seconds": 0.33539326100003564, "nonpadding_tokens": 9488, "peak_allocated_bytes": 14980128256, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.33828192400000034, "nonpadding_tokens": 7667, "peak_allocated_bytes": 14466909696, "peak_reserved_bytes": 16366174208}, {"compute_seconds": 0.335371379000037, "nonpadding_tokens": 12895, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3336930610000195, "nonpadding_tokens": 9931, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17553162240}, {"compute_seconds": 0.33835939199997256, "nonpadding_tokens": 10339, "peak_allocated_bytes": 14466570752, "peak_reserved_bytes": 17792237568}], "data_fetch_seconds": 0.00038566399996398104, "prefetch": {"data_wait_seconds": 5.397899998627054e-05, "consumed_batches": 4, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 554, "tokens": 50490833, "loss_tokens": 78811, "loss": 3.4060779110784027, "lr": 0.00029997502704001835, "optimizer_lrs": [0.019998335136001223, 0.00029997502704001835], "gradient_norm": 0.5504747033119202, "seconds": 0.2998135619999971, "nonpadding_tokens_per_second": 262866.6944692808, "ranks": [{"compute_seconds": 0.2955440300000305, "nonpadding_tokens": 7000, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18234736640}, {"compute_seconds": 0.29429805600000236, "nonpadding_tokens": 5512, "peak_allocated_bytes": 14466909696, "peak_reserved_bytes": 16326328320}, {"compute_seconds": 0.297065616999987, "nonpadding_tokens": 5302, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18280873984}, {"compute_seconds": 0.29713408800000707, "nonpadding_tokens": 2476, "peak_allocated_bytes": 14980128256, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.2936281920000283, "nonpadding_tokens": 4810, "peak_allocated_bytes": 14466909696, "peak_reserved_bytes": 16401825792}, {"compute_seconds": 0.2947390710000377, "nonpadding_tokens": 11554, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.29748753999996325, "nonpadding_tokens": 21310, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17553162240}, {"compute_seconds": 0.2943978299999799, "nonpadding_tokens": 20847, "peak_allocated_bytes": 14974088704, "peak_reserved_bytes": 17792237568}], "data_fetch_seconds": 0.00039413500002183355, "prefetch": {"data_wait_seconds": 6.340599999532515e-05, "consumed_batches": 5, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 555, "tokens": 50562766, "loss_tokens": 71933, "loss": 2.992895732834721, "lr": 0.0002999749090738612, "optimizer_lrs": [0.01999832727159075, 0.0002999749090738612], "gradient_norm": 0.7021991610527039, "seconds": 0.2984854619999737, "nonpadding_tokens_per_second": 240993.31176138265, "ranks": [{"compute_seconds": 0.2932610279999608, "nonpadding_tokens": 739, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.29531673399998226, "nonpadding_tokens": 14215, "peak_allocated_bytes": 14466909696, "peak_reserved_bytes": 16326328320}, {"compute_seconds": 0.29448258200000055, "nonpadding_tokens": 17850, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18280873984}, {"compute_seconds": 0.29457652899998266, "nonpadding_tokens": 11464, "peak_allocated_bytes": 14980128256, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.2913144919999695, "nonpadding_tokens": 5390, "peak_allocated_bytes": 14466909696, "peak_reserved_bytes": 16401825792}, {"compute_seconds": 0.2923253680000357, "nonpadding_tokens": 6331, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.29126174399999627, "nonpadding_tokens": 9867, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17553162240}, {"compute_seconds": 0.2960496709999916, "nonpadding_tokens": 6077, "peak_allocated_bytes": 14974088704, "peak_reserved_bytes": 17792237568}], "data_fetch_seconds": 0.00041475099999388476, "prefetch": {"data_wait_seconds": 7.26429999673428e-05, "consumed_batches": 6, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 556, "tokens": 50652529, "loss_tokens": 89763, "loss": 3.3879469408330825, "lr": 0.0002999747614776694, "optimizer_lrs": [0.019998317431844628, 0.0002999747614776694], "gradient_norm": 1.2709263563156128, "seconds": 0.2964690209999503, "nonpadding_tokens_per_second": 302773.62436466856, "ranks": [{"compute_seconds": 0.294192412999962, "nonpadding_tokens": 4257, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.29099170400002095, "nonpadding_tokens": 7632, "peak_allocated_bytes": 14466909696, "peak_reserved_bytes": 16326328320}, {"compute_seconds": 0.2935767029999852, "nonpadding_tokens": 11982, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18280873984}, {"compute_seconds": 0.2916425280000112, "nonpadding_tokens": 15957, "peak_allocated_bytes": 14980128256, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.2903709820000131, "nonpadding_tokens": 29709, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 16401825792}, {"compute_seconds": 0.2935240279999789, "nonpadding_tokens": 5469, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.290375197000003, "nonpadding_tokens": 5455, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17553162240}, {"compute_seconds": 0.29093062100002953, "nonpadding_tokens": 9302, "peak_allocated_bytes": 14974088704, "peak_reserved_bytes": 17792237568}], "data_fetch_seconds": 0.0003682770000068558, "prefetch": {"data_wait_seconds": 8.10129999564424e-05, "consumed_batches": 7, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 557, "tokens": 50745698, "loss_tokens": 93169, "loss": 2.96653661625648, "lr": 0.0002999746078232877, "optimizer_lrs": [0.01999830718821918, 0.0002999746078232877], "gradient_norm": 0.7914778590202332, "seconds": 0.31926571599996123, "nonpadding_tokens_per_second": 291822.752431117, "ranks": [{"compute_seconds": 0.316350492999959, "nonpadding_tokens": 7173, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3130167150000034, "nonpadding_tokens": 5502, "peak_allocated_bytes": 14466909696, "peak_reserved_bytes": 16326328320}, {"compute_seconds": 0.31387910500001226, "nonpadding_tokens": 3278, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.31640186200002063, "nonpadding_tokens": 10872, "peak_allocated_bytes": 14980128256, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.3167231950000087, "nonpadding_tokens": 27393, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 16401825792}, {"compute_seconds": 0.315587794999999, "nonpadding_tokens": 17948, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31191319200002, "nonpadding_tokens": 10194, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17553162240}, {"compute_seconds": 0.3169181970000068, "nonpadding_tokens": 10809, "peak_allocated_bytes": 14974088704, "peak_reserved_bytes": 17792237568}], "data_fetch_seconds": 0.00038636499999711305, "prefetch": {"data_wait_seconds": 9.105299996292615e-05, "consumed_batches": 8, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 558, "tokens": 50829205, "loss_tokens": 83507, "loss": 3.097574155460021, "lr": 0.00029997446970723854, "optimizer_lrs": [0.019998297980482572, 0.00029997446970723854], "gradient_norm": 0.6167598962783813, "seconds": 0.3926811819999898, "nonpadding_tokens_per_second": 212658.52255686183, "ranks": [{"compute_seconds": 0.390221256000018, "nonpadding_tokens": 15928, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3864555569999766, "nonpadding_tokens": 10884, "peak_allocated_bytes": 14466909696, "peak_reserved_bytes": 16326328320}, {"compute_seconds": 0.38879707400002417, "nonpadding_tokens": 18717, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3890825130000053, "nonpadding_tokens": 15565, "peak_allocated_bytes": 14980128256, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.3897902890000182, "nonpadding_tokens": 3163, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 16401825792}, {"compute_seconds": 0.38719464300004347, "nonpadding_tokens": 5185, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.38536529300000666, "nonpadding_tokens": 5400, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17553162240}, {"compute_seconds": 0.39036046199998964, "nonpadding_tokens": 8665, "peak_allocated_bytes": 14974088704, "peak_reserved_bytes": 17792237568}], "data_fetch_seconds": 0.00035886799997797425, "prefetch": {"data_wait_seconds": 0.00010021699996798361, "consumed_batches": 9, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 559, "tokens": 50915037, "loss_tokens": 85832, "loss": 2.952301443517569, "lr": 0.00029997432735540217, "optimizer_lrs": [0.019998288490360144, 0.00029997432735540217], "gradient_norm": 0.4509032368659973, "seconds": 0.4444559549999667, "nonpadding_tokens_per_second": 193116.9985111493, "ranks": [{"compute_seconds": 0.44160370800000237, "nonpadding_tokens": 9266, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.43892626999996764, "nonpadding_tokens": 18779, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18473811968}, {"compute_seconds": 0.4414610629999629, "nonpadding_tokens": 9863, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.43998923700002024, "nonpadding_tokens": 13263, "peak_allocated_bytes": 14982032896, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.43905256599998665, "nonpadding_tokens": 3530, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 16401825792}, {"compute_seconds": 0.4409818910000354, "nonpadding_tokens": 19245, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.4421785250000312, "nonpadding_tokens": 7839, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17553162240}, {"compute_seconds": 0.438947010999982, "nonpadding_tokens": 4047, "peak_allocated_bytes": 14974088704, "peak_reserved_bytes": 17792237568}], "data_fetch_seconds": 0.0003830059999927471, "prefetch": {"data_wait_seconds": 0.00010996099997839792, "consumed_batches": 10, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 560, "tokens": 50984762, "loss_tokens": 69725, "loss": 3.231114422732162, "lr": 0.00029997421142562494, "optimizer_lrs": [0.019998280761708332, 0.00029997421142562494], "gradient_norm": 0.6036185622215271, "seconds": 0.2858105720000026, "nonpadding_tokens_per_second": 243955.2865805096, "ranks": [{"compute_seconds": 0.20987420500000553, "nonpadding_tokens": 5490, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.2321227919999842, "nonpadding_tokens": 9623, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18473811968}, {"compute_seconds": 0.2344179070000223, "nonpadding_tokens": 9731, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.23494825199998104, "nonpadding_tokens": 9513, "peak_allocated_bytes": 14982032896, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.23587995599996248, "nonpadding_tokens": 13287, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.2326737789999811, "nonpadding_tokens": 7590, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.23188509199997043, "nonpadding_tokens": 6987, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17553162240}, {"compute_seconds": 0.23166148500001782, "nonpadding_tokens": 7504, "peak_allocated_bytes": 14974088704, "peak_reserved_bytes": 17792237568}], "data_fetch_seconds": 0.01940388099995971, "prefetch": {"data_wait_seconds": 0.00011829900002169325, "consumed_batches": 11, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 561, "tokens": 51049274, "loss_tokens": 64512, "loss": 3.3671150813027033, "lr": 0.00029997410393077354, "optimizer_lrs": [0.019998273595384905, 0.00029997410393077354], "gradient_norm": 0.4276994466781616, "seconds": 0.3178912100000275, "nonpadding_tokens_per_second": 202937.3507999621, "ranks": [{"compute_seconds": 0.3133019690000083, "nonpadding_tokens": 7159, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3115153740000096, "nonpadding_tokens": 7059, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18473811968}, {"compute_seconds": 0.3113864329999956, "nonpadding_tokens": 18758, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3143143719999557, "nonpadding_tokens": 14062, "peak_allocated_bytes": 14982032896, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.3104701390000173, "nonpadding_tokens": 5538, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.31276200000002063, "nonpadding_tokens": 3005, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31009270699996705, "nonpadding_tokens": 2517, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.3155989960000056, "nonpadding_tokens": 6414, "peak_allocated_bytes": 14974088704, "peak_reserved_bytes": 17792237568}], "data_fetch_seconds": 0.0003524559999732446, "prefetch": {"data_wait_seconds": 0.00012741199999766195, "consumed_batches": 12, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 562, "tokens": 51154932, "loss_tokens": 105658, "loss": 3.6237379682560715, "lr": 0.00029997392739240683, "optimizer_lrs": [0.019998261826160456, 0.00029997392739240683], "gradient_norm": 0.9459865093231201, "seconds": 0.3770716849999758, "nonpadding_tokens_per_second": 280206.66680397064, "ranks": [{"compute_seconds": 0.37266979000003175, "nonpadding_tokens": 18377, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.37446718600000395, "nonpadding_tokens": 4473, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18473811968}, {"compute_seconds": 0.37133785800000396, "nonpadding_tokens": 6077, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.37156835199999705, "nonpadding_tokens": 9310, "peak_allocated_bytes": 14982032896, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.37054372099999, "nonpadding_tokens": 22587, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3731380769999646, "nonpadding_tokens": 19322, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.37433325500001047, "nonpadding_tokens": 13098, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.3747412380000128, "nonpadding_tokens": 12414, "peak_allocated_bytes": 14974088704, "peak_reserved_bytes": 17792237568}], "data_fetch_seconds": 0.00034856100000979495, "prefetch": {"data_wait_seconds": 0.00013680799999349347, "consumed_batches": 13, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 563, "tokens": 51271342, "loss_tokens": 116410, "loss": 3.2247336998539646, "lr": 0.0002999737321948077, "optimizer_lrs": [0.01999824881298718, 0.0002999737321948077], "gradient_norm": 0.3917027711868286, "seconds": 0.33901589499998863, "nonpadding_tokens_per_second": 343376.23019122425, "ranks": [{"compute_seconds": 0.33462687399998003, "nonpadding_tokens": 20078, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3329769740000188, "nonpadding_tokens": 10466, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18473811968}, {"compute_seconds": 0.33217887400002155, "nonpadding_tokens": 13418, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3357288049999738, "nonpadding_tokens": 7592, "peak_allocated_bytes": 14982032896, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.33657525600000326, "nonpadding_tokens": 14017, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3342542279999634, "nonpadding_tokens": 14602, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.33585686399999304, "nonpadding_tokens": 24795, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.33669263699999874, "nonpadding_tokens": 11442, "peak_allocated_bytes": 14976999936, "peak_reserved_bytes": 17794334720}], "data_fetch_seconds": 0.0003538420000381848, "prefetch": {"data_wait_seconds": 0.00014509699998654924, "consumed_batches": 14, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 564, "tokens": 51360443, "loss_tokens": 89101, "loss": 2.8855342181905925, "lr": 0.0002999735822974592, "optimizer_lrs": [0.019998238819830615, 0.0002999735822974592], "gradient_norm": 0.6794210076332092, "seconds": 0.3295210549999865, "nonpadding_tokens_per_second": 270395.46835635026, "ranks": [{"compute_seconds": 0.3271641240000349, "nonpadding_tokens": 16527, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3267068750000135, "nonpadding_tokens": 22684, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18473811968}, {"compute_seconds": 0.32353268299999627, "nonpadding_tokens": 18553, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.32648469800000157, "nonpadding_tokens": 2972, "peak_allocated_bytes": 14982032896, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.32303148200003307, "nonpadding_tokens": 1843, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3254616520000013, "nonpadding_tokens": 16254, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3231256960000337, "nonpadding_tokens": 5152, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.3232567270000004, "nonpadding_tokens": 5116, "peak_allocated_bytes": 14976999936, "peak_reserved_bytes": 17794334720}], "data_fetch_seconds": 0.0003765380000118057, "prefetch": {"data_wait_seconds": 0.00015483999995069553, "consumed_batches": 15, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 565, "tokens": 51445280, "loss_tokens": 84837, "loss": 3.479982201162229, "lr": 0.0002999734391772547, "optimizer_lrs": [0.01999822927848365, 0.0002999734391772547], "gradient_norm": 0.550309419631958, "seconds": 0.29691131499998846, "nonpadding_tokens_per_second": 285731.7849271029, "ranks": [{"compute_seconds": 0.29301278299999467, "nonpadding_tokens": 5659, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.2939745969999876, "nonpadding_tokens": 11254, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.2908083540000348, "nonpadding_tokens": 11412, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.2930851890000099, "nonpadding_tokens": 10647, "peak_allocated_bytes": 14982032896, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.2939662710000448, "nonpadding_tokens": 7092, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.2923972169999729, "nonpadding_tokens": 16054, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.29391639400000713, "nonpadding_tokens": 5980, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.2945724909999967, "nonpadding_tokens": 16739, "peak_allocated_bytes": 14976999936, "peak_reserved_bytes": 17794334720}], "data_fetch_seconds": 0.00033726800000977164, "prefetch": {"data_wait_seconds": 0.00016397899999276433, "consumed_batches": 16, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 566, "tokens": 51529974, "loss_tokens": 84694, "loss": 3.0927083825300494, "lr": 0.00029997329591267006, "optimizer_lrs": [0.019998219727511337, 0.00029997329591267006], "gradient_norm": 0.4251135289669037, "seconds": 0.33359117200001265, "nonpadding_tokens_per_second": 253885.61541429756, "ranks": [{"compute_seconds": 0.3303708619999952, "nonpadding_tokens": 8108, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3283709309999949, "nonpadding_tokens": 9625, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.32603697799999054, "nonpadding_tokens": 11215, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.33121132400003717, "nonpadding_tokens": 14518, "peak_allocated_bytes": 14982032896, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.3254679970000325, "nonpadding_tokens": 11784, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.32900127400000656, "nonpadding_tokens": 15228, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.32492727999999715, "nonpadding_tokens": 8885, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.32555560200000855, "nonpadding_tokens": 5331, "peak_allocated_bytes": 14976999936, "peak_reserved_bytes": 17794334720}], "data_fetch_seconds": 0.0004005609999921944, "prefetch": {"data_wait_seconds": 0.00017406899998206882, "consumed_batches": 17, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 567, "tokens": 51587946, "loss_tokens": 57972, "loss": 3.320635932001656, "lr": 0.000299973197627701, "optimizer_lrs": [0.01999821317518007, 0.000299973197627701], "gradient_norm": 0.4710610806941986, "seconds": 0.342267000999982, "nonpadding_tokens_per_second": 169376.53887353005, "ranks": [{"compute_seconds": 0.3396200060000183, "nonpadding_tokens": 3071, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.33808057800001734, "nonpadding_tokens": 7734, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.335126884000033, "nonpadding_tokens": 15591, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.33859299799996734, "nonpadding_tokens": 10030, "peak_allocated_bytes": 14982032896, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.3339802529999929, "nonpadding_tokens": 12011, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3366927510000437, "nonpadding_tokens": 1632, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3340027720000194, "nonpadding_tokens": 3385, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.33453267999999525, "nonpadding_tokens": 4518, "peak_allocated_bytes": 14976999936, "peak_reserved_bytes": 17794334720}], "data_fetch_seconds": 0.00042386099994473625, "prefetch": {"data_wait_seconds": 0.0001850349999727996, "consumed_batches": 18, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 568, "tokens": 51652439, "loss_tokens": 64493, "loss": 3.3650672359790983, "lr": 0.0002999730880749933, "optimizer_lrs": [0.01999820587166622, 0.0002999730880749933], "gradient_norm": 3.7639946937561035, "seconds": 0.3399082870000143, "nonpadding_tokens_per_second": 189736.47441551575, "ranks": [{"compute_seconds": 0.3358908809999548, "nonpadding_tokens": 5191, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.33632799800000157, "nonpadding_tokens": 8718, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.33573110700001507, "nonpadding_tokens": 5360, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3340172609999854, "nonpadding_tokens": 10697, "peak_allocated_bytes": 14982032896, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.33696273000003885, "nonpadding_tokens": 16682, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.33447015999996665, "nonpadding_tokens": 11876, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.33248693000001595, "nonpadding_tokens": 4176, "peak_allocated_bytes": 14976931328, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.33350461300000234, "nonpadding_tokens": 1793, "peak_allocated_bytes": 14976999936, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.00042755300000862917, "prefetch": {"data_wait_seconds": 0.00019495799996605, "consumed_batches": 19, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 569, "tokens": 51725339, "loss_tokens": 72900, "loss": 3.3872663751714676, "lr": 0.0002999729639725181, "optimizer_lrs": [0.019998197598167875, 0.0002999729639725181], "gradient_norm": 0.46120116114616394, "seconds": 0.7066869419999762, "nonpadding_tokens_per_second": 103157.41761647332, "ranks": [{"compute_seconds": 0.7018264929999987, "nonpadding_tokens": 2207, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.70227569299999, "nonpadding_tokens": 12387, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.6990283340000474, "nonpadding_tokens": 5751, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.7012177189999989, "nonpadding_tokens": 8779, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.6982876530000226, "nonpadding_tokens": 7593, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.7008757930000229, "nonpadding_tokens": 9145, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.7043059779999794, "nonpadding_tokens": 14051, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.700499394000019, "nonpadding_tokens": 12987, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.0003940249999914158, "prefetch": {"data_wait_seconds": 0.0002033279999551496, "consumed_batches": 20, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 570, "tokens": 51821421, "loss_tokens": 96082, "loss": 3.2212712839033326, "lr": 0.0002999727999697422, "optimizer_lrs": [0.019998186664649482, 0.0002999727999697422], "gradient_norm": 0.605654776096344, "seconds": 0.34780754100000877, "nonpadding_tokens_per_second": 276250.4795719699, "ranks": [{"compute_seconds": 0.3434775409999702, "nonpadding_tokens": 13331, "peak_allocated_bytes": 14993425920, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.34217451600000004, "nonpadding_tokens": 7211, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3428201429999831, "nonpadding_tokens": 5828, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3453208459999928, "nonpadding_tokens": 12484, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.345468134999976, "nonpadding_tokens": 15768, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3443247730000394, "nonpadding_tokens": 14386, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3413487799999757, "nonpadding_tokens": 15726, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.34217334400000254, "nonpadding_tokens": 11348, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.0003797080000254027, "prefetch": {"data_wait_seconds": 0.0002124639999721012, "consumed_batches": 21, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 571, "tokens": 51930764, "loss_tokens": 109343, "loss": 3.12433894945264, "lr": 0.00029997261272845296, "optimizer_lrs": [0.019998174181896866, 0.00029997261272845296], "gradient_norm": 0.4820402264595032, "seconds": 0.37491966099997853, "nonpadding_tokens_per_second": 291643.8143264145, "ranks": [{"compute_seconds": 0.3719282280000016, "nonpadding_tokens": 23010, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.36866518499999756, "nonpadding_tokens": 16372, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3692941689999998, "nonpadding_tokens": 4162, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3698193779999883, "nonpadding_tokens": 13989, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.3725785689999839, "nonpadding_tokens": 1679, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.37086921899998515, "nonpadding_tokens": 13057, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3721021549999932, "nonpadding_tokens": 18138, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.368674446, "nonpadding_tokens": 18936, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.0003426960000183499, "prefetch": {"data_wait_seconds": 0.0002211879999691746, "consumed_batches": 22, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 572, "tokens": 52028368, "loss_tokens": 97604, "loss": 3.107346714786279, "lr": 0.0002999724450467984, "optimizer_lrs": [0.019998163003119896, 0.0002999724450467984], "gradient_norm": 0.4367014765739441, "seconds": 0.2973749610000027, "nonpadding_tokens_per_second": 328218.6222800349, "ranks": [{"compute_seconds": 0.29246957000003704, "nonpadding_tokens": 5212, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.2948512019999612, "nonpadding_tokens": 12021, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.2937106139999628, "nonpadding_tokens": 14991, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.29406225200000335, "nonpadding_tokens": 16987, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.29436590400001705, "nonpadding_tokens": 15868, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.2919132540000078, "nonpadding_tokens": 15609, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.29424181599995336, "nonpadding_tokens": 5729, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.29503455700000814, "nonpadding_tokens": 11187, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.0003739299999665491, "prefetch": {"data_wait_seconds": 0.00023105099997167144, "consumed_batches": 23, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 573, "tokens": 52104163, "loss_tokens": 75795, "loss": 3.189564161554192, "lr": 0.0002999723144795805, "optimizer_lrs": [0.019998154298638705, 0.0002999723144795805], "gradient_norm": 0.5874981880187988, "seconds": 0.3879614530000026, "nonpadding_tokens_per_second": 195367.34748748218, "ranks": [{"compute_seconds": 0.384030537000001, "nonpadding_tokens": 24950, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3808283849999725, "nonpadding_tokens": 2318, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3811047570000028, "nonpadding_tokens": 3765, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.384042533000013, "nonpadding_tokens": 5582, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.38035840300000245, "nonpadding_tokens": 15910, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.38300057599997217, "nonpadding_tokens": 3683, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3812860060000389, "nonpadding_tokens": 7786, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.3854859079999642, "nonpadding_tokens": 11801, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.00034698999996862767, "prefetch": {"data_wait_seconds": 0.0002403559999635263, "consumed_batches": 24, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 574, "tokens": 52172670, "loss_tokens": 68507, "loss": 3.39964962522078, "lr": 0.00029997219620143685, "optimizer_lrs": [0.019998146413429126, 0.00029997219620143685], "gradient_norm": 0.6236481666564941, "seconds": 0.365302985000028, "nonpadding_tokens_per_second": 187534.73914261814, "ranks": [{"compute_seconds": 0.36267943699999705, "nonpadding_tokens": 4288, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3618587739999839, "nonpadding_tokens": 16265, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3613087190000215, "nonpadding_tokens": 2907, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.36125732499999685, "nonpadding_tokens": 13398, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.3580697790000045, "nonpadding_tokens": 4613, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.36059018899999273, "nonpadding_tokens": 10350, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.357854298999996, "nonpadding_tokens": 5898, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.362392041000021, "nonpadding_tokens": 10788, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.00041615099996761273, "prefetch": {"data_wait_seconds": 0.00025092299996742895, "consumed_batches": 25, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 575, "tokens": 52260687, "loss_tokens": 88017, "loss": 3.0256484542758786, "lr": 0.00029997204386904703, "optimizer_lrs": [0.01999813625793647, 0.00029997204386904703], "gradient_norm": 0.4525589644908905, "seconds": 0.3124429710000527, "nonpadding_tokens_per_second": 281705.8092818646, "ranks": [{"compute_seconds": 0.3078668360000165, "nonpadding_tokens": 10222, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3058071959999893, "nonpadding_tokens": 6280, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.30847909400000617, "nonpadding_tokens": 23200, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.30577544600004103, "nonpadding_tokens": 13898, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.30973452000000634, "nonpadding_tokens": 4494, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3064941189999786, "nonpadding_tokens": 9940, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3091289619999884, "nonpadding_tokens": 9497, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.3097340190000182, "nonpadding_tokens": 10486, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.00039905400001316593, "prefetch": {"data_wait_seconds": 0.0002604429999593094, "consumed_batches": 26, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 576, "tokens": 52347698, "loss_tokens": 87011, "loss": 2.9590820413510937, "lr": 0.0002999718928687446, "optimizer_lrs": [0.019998126191249643, 0.0002999718928687446], "gradient_norm": 0.5445228815078735, "seconds": 0.2760286810000139, "nonpadding_tokens_per_second": 315224.4892986161, "ranks": [{"compute_seconds": 0.2716356450000035, "nonpadding_tokens": 12040, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.2697799219999979, "nonpadding_tokens": 1515, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.27272395500000357, "nonpadding_tokens": 20002, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.27369029599998385, "nonpadding_tokens": 5846, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.26966309900001306, "nonpadding_tokens": 9434, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.2709742949999736, "nonpadding_tokens": 18129, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2733756490000019, "nonpadding_tokens": 11184, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.2702255880000166, "nonpadding_tokens": 8861, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.0004262239999661688, "prefetch": {"data_wait_seconds": 0.0002712239999738131, "consumed_batches": 27, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 577, "tokens": 52443939, "loss_tokens": 96241, "loss": 3.0083371432133914, "lr": 0.000299971725376894, "optimizer_lrs": [0.01999811502512627, 0.000299971725376894], "gradient_norm": 0.4401289224624634, "seconds": 0.348368569999991, "nonpadding_tokens_per_second": 276262.0060701874, "ranks": [{"compute_seconds": 0.34168135800001664, "nonpadding_tokens": 13152, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3447418529999595, "nonpadding_tokens": 22599, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3414781719999951, "nonpadding_tokens": 11947, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.34500615700000026, "nonpadding_tokens": 3278, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.34096905400002697, "nonpadding_tokens": 4713, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.34323011200001474, "nonpadding_tokens": 3507, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.34569945600003393, "nonpadding_tokens": 17669, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.3459813100000133, "nonpadding_tokens": 19376, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.00037454400001024624, "prefetch": {"data_wait_seconds": 0.0002811149999644158, "consumed_batches": 28, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 578, "tokens": 52553619, "loss_tokens": 109680, "loss": 2.8491566374908825, "lr": 0.00029997153389007235, "optimizer_lrs": [0.01999810225933816, 0.00029997153389007235], "gradient_norm": 0.4196028709411621, "seconds": 0.3590460949999965, "nonpadding_tokens_per_second": 305476.09771386336, "ranks": [{"compute_seconds": 0.3554930450000029, "nonpadding_tokens": 14128, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3524564119999809, "nonpadding_tokens": 18518, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.35492748399997254, "nonpadding_tokens": 9660, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.35540882699996246, "nonpadding_tokens": 11960, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17792237568}, {"compute_seconds": 0.35106661600002553, "nonpadding_tokens": 13009, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.35326427300003616, "nonpadding_tokens": 13399, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.35580855799997835, "nonpadding_tokens": 9654, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17593008128}, {"compute_seconds": 0.3567285819999597, "nonpadding_tokens": 19352, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.000366380000002664, "prefetch": {"data_wait_seconds": 0.00028949599995087283, "consumed_batches": 29, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 579, "tokens": 52659762, "loss_tokens": 106143, "loss": 2.9401082148610835, "lr": 0.00029997134796314956, "optimizer_lrs": [0.01999808986420997, 0.00029997134796314956], "gradient_norm": 0.4199081361293793, "seconds": 0.3328882280000016, "nonpadding_tokens_per_second": 318854.77187856426, "ranks": [{"compute_seconds": 0.32908899299997074, "nonpadding_tokens": 2280, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3269646380000495, "nonpadding_tokens": 22710, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.32931883100002324, "nonpadding_tokens": 3086, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3280506290000176, "nonpadding_tokens": 2912, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3302630850000128, "nonpadding_tokens": 29539, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.32786495799996374, "nonpadding_tokens": 28034, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.33044757199996866, "nonpadding_tokens": 2627, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3270121069999732, "nonpadding_tokens": 14955, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.0004016890000002604, "prefetch": {"data_wait_seconds": 0.0002986929999480026, "consumed_batches": 30, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 580, "tokens": 52722608, "loss_tokens": 62846, "loss": 2.9072086926773384, "lr": 0.00029997123759281793, "optimizer_lrs": [0.019998082506187863, 0.00029997123759281793], "gradient_norm": 0.5673766732215881, "seconds": 0.31991473499999756, "nonpadding_tokens_per_second": 196446.0936755554, "ranks": [{"compute_seconds": 0.31691884700001083, "nonpadding_tokens": 8786, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.31654782000003934, "nonpadding_tokens": 17084, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3133398279999824, "nonpadding_tokens": 1582, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3161448419999715, "nonpadding_tokens": 13360, "peak_allocated_bytes": 14984796672, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31695028600000796, "nonpadding_tokens": 1824, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3155219530000295, "nonpadding_tokens": 3285, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31224836100000175, "nonpadding_tokens": 958, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3174451349999572, "nonpadding_tokens": 15967, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.0003721759999848473, "prefetch": {"data_wait_seconds": 0.0003060439999700293, "consumed_batches": 31, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 581, "tokens": 52821218, "loss_tokens": 98610, "loss": 3.395473963086908, "lr": 0.0002999710639860535, "optimizer_lrs": [0.019998070932403567, 0.0002999710639860535], "gradient_norm": 0.49562305212020874, "seconds": 0.6302503130000332, "nonpadding_tokens_per_second": 156461.6438357799, "ranks": [{"compute_seconds": 0.6264095249999855, "nonpadding_tokens": 17687, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.6278976870000292, "nonpadding_tokens": 23068, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.6244487360000335, "nonpadding_tokens": 16605, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.62512731999999, "nonpadding_tokens": 25093, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.6236344139999801, "nonpadding_tokens": 1853, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.6250818139999978, "nonpadding_tokens": 2208, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.6276002989999938, "nonpadding_tokens": 2689, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.6241419680000035, "nonpadding_tokens": 9407, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.00045380900002101043, "prefetch": {"data_wait_seconds": 0.0003158849999778113, "consumed_batches": 32, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 582, "tokens": 52913441, "loss_tokens": 92223, "loss": 2.8180256687594203, "lr": 0.0002999709011511974, "optimizer_lrs": [0.019998060076746493, 0.0002999709011511974], "gradient_norm": 0.4301034212112427, "seconds": 0.24771174800002882, "nonpadding_tokens_per_second": 372299.661782651, "ranks": [{"compute_seconds": 0.24540496200000916, "nonpadding_tokens": 15336, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.2446840929999894, "nonpadding_tokens": 3022, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24186440700003686, "nonpadding_tokens": 9131, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.24457913200001258, "nonpadding_tokens": 8855, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24089789399999972, "nonpadding_tokens": 14441, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.24397774400000571, "nonpadding_tokens": 15763, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2407097469999826, "nonpadding_tokens": 19763, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.24536524000001236, "nonpadding_tokens": 5912, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.00034261600001173065, "prefetch": {"data_wait_seconds": 0.0003247489999722575, "consumed_batches": 33, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 583, "tokens": 53001553, "loss_tokens": 88112, "loss": 2.4149764646132197, "lr": 0.00029997074514824415, "optimizer_lrs": [0.019998049676549612, 0.00029997074514824415], "gradient_norm": 0.43225622177124023, "seconds": 0.21264770700003055, "nonpadding_tokens_per_second": 414356.6899594518, "ranks": [{"compute_seconds": 0.2098998550000033, "nonpadding_tokens": 4452, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.2098927339999932, "nonpadding_tokens": 9122, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20911849200001598, "nonpadding_tokens": 1303, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.20756049300001678, "nonpadding_tokens": 18984, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2060668309999869, "nonpadding_tokens": 18590, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.2076010270000097, "nonpadding_tokens": 11318, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.20582788799998752, "nonpadding_tokens": 11951, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.21026380199998584, "nonpadding_tokens": 12392, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.00045940900002960916, "prefetch": {"data_wait_seconds": 0.000334364999972081, "consumed_batches": 34, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 584, "tokens": 53106910, "loss_tokens": 105357, "loss": 3.1790546071926875, "lr": 0.00029997055806546373, "optimizer_lrs": [0.01999803720436425, 0.00029997055806546373], "gradient_norm": 0.6243709921836853, "seconds": 0.2866634609999892, "nonpadding_tokens_per_second": 367528.5285138031, "ranks": [{"compute_seconds": 0.28358568999999534, "nonpadding_tokens": 3857, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.283952768000006, "nonpadding_tokens": 19509, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28179006300001674, "nonpadding_tokens": 20260, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.2836074040000085, "nonpadding_tokens": 10581, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28392536400002655, "nonpadding_tokens": 19586, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.2826052760000266, "nonpadding_tokens": 9211, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.28369190100005426, "nonpadding_tokens": 17890, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.2843750659999955, "nonpadding_tokens": 4463, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.0003569760000345923, "prefetch": {"data_wait_seconds": 0.0003426859999677845, "consumed_batches": 35, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 585, "tokens": 53209036, "loss_tokens": 102126, "loss": 2.48516171200282, "lr": 0.0002999703761509067, "optimizer_lrs": [0.019998025076727113, 0.0002999703761509067], "gradient_norm": 1.804027795791626, "seconds": 0.35082957499997747, "nonpadding_tokens_per_second": 291098.60535562475, "ranks": [{"compute_seconds": 0.3461395209999978, "nonpadding_tokens": 3545, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.34454273599999397, "nonpadding_tokens": 6582, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3469397229999913, "nonpadding_tokens": 3224, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.34755694699998685, "nonpadding_tokens": 5868, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34826695999998947, "nonpadding_tokens": 10832, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3463555189999852, "nonpadding_tokens": 25151, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3438326519999464, "nonpadding_tokens": 18823, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3485251749999634, "nonpadding_tokens": 28101, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.00038049900001624337, "prefetch": {"data_wait_seconds": 0.0003517699999662227, "consumed_batches": 36, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 586, "tokens": 53316147, "loss_tokens": 107111, "loss": 1.750337558700787, "lr": 0.00029997018475479365, "optimizer_lrs": [0.019998012316986245, 0.00029997018475479365], "gradient_norm": 4.216708183288574, "seconds": 0.409439003999978, "nonpadding_tokens_per_second": 261604.2901472224, "ranks": [{"compute_seconds": 0.406504899999959, "nonpadding_tokens": 23928, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.4031048589999955, "nonpadding_tokens": 18291, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4034715629999823, "nonpadding_tokens": 12140, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.4066119409999942, "nonpadding_tokens": 18735, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4028356629999621, "nonpadding_tokens": 9865, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.40545313699999497, "nonpadding_tokens": 2628, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.4025963690000367, "nonpadding_tokens": 10752, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.407143657000006, "nonpadding_tokens": 10772, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17832083456}], "data_fetch_seconds": 0.0003626850000273407, "prefetch": {"data_wait_seconds": 0.0003615059999475534, "consumed_batches": 37, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 587, "tokens": 53440842, "loss_tokens": 124695, "loss": 1.6840581017683147, "lr": 0.000299969961161631, "optimizer_lrs": [0.0199979974107754, 0.000299969961161631], "gradient_norm": 3.5076355934143066, "seconds": 0.3709505700000477, "nonpadding_tokens_per_second": 336149.90805913566, "ranks": [{"compute_seconds": 0.3683521990000145, "nonpadding_tokens": 9933, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18236833792}, {"compute_seconds": 0.3675620630000367, "nonpadding_tokens": 10688, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3671905399999673, "nonpadding_tokens": 10428, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3653238299999657, "nonpadding_tokens": 22829, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.363542974999973, "nonpadding_tokens": 25464, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.36702903600001946, "nonpadding_tokens": 11074, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3634148660000278, "nonpadding_tokens": 24837, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.36823524100003624, "nonpadding_tokens": 9442, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004014390000293133, "prefetch": {"data_wait_seconds": 0.0003707809999724532, "consumed_batches": 38, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 588, "tokens": 53565704, "loss_tokens": 124862, "loss": 2.072342365972033, "lr": 0.00029996973643216604, "optimizer_lrs": [0.01999798242881107, 0.00029996973643216604], "gradient_norm": 1.161860704421997, "seconds": 0.33490601099998685, "nonpadding_tokens_per_second": 372826.9899581017, "ranks": [{"compute_seconds": 0.33219528900002615, "nonpadding_tokens": 1707, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.33249116899997944, "nonpadding_tokens": 9425, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3298785389999921, "nonpadding_tokens": 19922, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.33254012300000113, "nonpadding_tokens": 29277, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3293258900000069, "nonpadding_tokens": 24564, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.331521218999967, "nonpadding_tokens": 17959, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.32893998899999133, "nonpadding_tokens": 3780, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3293505440000217, "nonpadding_tokens": 18228, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004060409999624426, "prefetch": {"data_wait_seconds": 0.0003798219999708863, "consumed_batches": 39, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 589, "tokens": 53659319, "loss_tokens": 93615, "loss": 3.0653314105645464, "lr": 0.00029996956739247553, "optimizer_lrs": [0.01999797115949837, 0.00029996956739247553], "gradient_norm": 0.46485522389411926, "seconds": 0.3915161710000348, "nonpadding_tokens_per_second": 239108.89749683338, "ranks": [{"compute_seconds": 0.3858108250000214, "nonpadding_tokens": 13746, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3884944639999617, "nonpadding_tokens": 5322, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38565133299999843, "nonpadding_tokens": 4503, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3856533309999577, "nonpadding_tokens": 16217, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.38393250900003295, "nonpadding_tokens": 4254, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3867550680000136, "nonpadding_tokens": 11812, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.38454457699998557, "nonpadding_tokens": 18279, "peak_allocated_bytes": 14982388224, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3891402410000069, "nonpadding_tokens": 19482, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003418000000010579, "prefetch": {"data_wait_seconds": 0.00038913499999182477, "consumed_batches": 40, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 590, "tokens": 53782910, "loss_tokens": 123591, "loss": 2.59222733653745, "lr": 0.00029996934350446416, "optimizer_lrs": [0.019997956233630945, 0.00029996934350446416], "gradient_norm": 0.4068358838558197, "seconds": 0.36962331800003767, "nonpadding_tokens_per_second": 334370.13841206685, "ranks": [{"compute_seconds": 0.36732319800000823, "nonpadding_tokens": 11948, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3639829440000426, "nonpadding_tokens": 27925, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36643899699998883, "nonpadding_tokens": 8451, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.36461392500001466, "nonpadding_tokens": 14378, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.36317895700000236, "nonpadding_tokens": 9200, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3663602400000059, "nonpadding_tokens": 25791, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.36316056200001867, "nonpadding_tokens": 10976, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3636935279999989, "nonpadding_tokens": 14922, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003647919999707483, "prefetch": {"data_wait_seconds": 0.00039833999996119474, "consumed_batches": 41, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 591, "tokens": 53905793, "loss_tokens": 122883, "loss": 2.8342132353539546, "lr": 0.00029996912008560224, "optimizer_lrs": [0.019997941339040152, 0.00029996912008560224], "gradient_norm": 0.5752902626991272, "seconds": 0.24234089200001563, "nonpadding_tokens_per_second": 507066.7149314284, "ranks": [{"compute_seconds": 0.2387151480000398, "nonpadding_tokens": 7562, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2364734039999803, "nonpadding_tokens": 9223, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2368686240000102, "nonpadding_tokens": 16406, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.2371111209999981, "nonpadding_tokens": 18378, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23572519399999692, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.23896384500000067, "nonpadding_tokens": 13417, "peak_allocated_bytes": 14967503360, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.24000097499998674, "nonpadding_tokens": 11767, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.2366289790000451, "nonpadding_tokens": 13362, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003990690000250652, "prefetch": {"data_wait_seconds": 0.000406735999959551, "consumed_batches": 42, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 592, "tokens": 54034032, "loss_tokens": 128239, "loss": 3.4581426087227753, "lr": 0.00029996888606389756, "optimizer_lrs": [0.01999792573759317, 0.00029996888606389756], "gradient_norm": 0.3775699734687805, "seconds": 0.3621654550000244, "nonpadding_tokens_per_second": 354089.54175375827, "ranks": [{"compute_seconds": 0.35178212399995346, "nonpadding_tokens": 22143, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.35960833299998285, "nonpadding_tokens": 15542, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3568525910000062, "nonpadding_tokens": 19132, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.35984483400000045, "nonpadding_tokens": 11463, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3561070399999835, "nonpadding_tokens": 17440, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3573357660000056, "nonpadding_tokens": 18161, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.355991696999979, "nonpadding_tokens": 17006, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3566431329999773, "nonpadding_tokens": 7352, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000365102999978717, "prefetch": {"data_wait_seconds": 0.0004144709999565066, "consumed_batches": 43, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 593, "tokens": 54141929, "loss_tokens": 107897, "loss": 2.718418086230386, "lr": 0.0002999686884797951, "optimizer_lrs": [0.019997912565319676, 0.0002999686884797951], "gradient_norm": 0.4685993492603302, "seconds": 0.33247718100000156, "nonpadding_tokens_per_second": 324524.52729379793, "ranks": [{"compute_seconds": 0.3291261059999897, "nonpadding_tokens": 10965, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3269607990000054, "nonpadding_tokens": 21015, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3266983959999834, "nonpadding_tokens": 10816, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.328054399999985, "nonpadding_tokens": 14663, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3260818970000514, "nonpadding_tokens": 18393, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.32883361700004343, "nonpadding_tokens": 6071, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.33002239099999997, "nonpadding_tokens": 6574, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.32678507699995407, "nonpadding_tokens": 19400, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040436300002966163, "prefetch": {"data_wait_seconds": 0.00042391499995630966, "consumed_batches": 44, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 594, "tokens": 54292406, "loss_tokens": 150477, "loss": 2.8804617981485543, "lr": 0.0002999684118778168, "optimizer_lrs": [0.01999789412518779, 0.0002999684118778168], "gradient_norm": 1.013907790184021, "seconds": 0.33706732499996406, "nonpadding_tokens_per_second": 446430.1011675221, "ranks": [{"compute_seconds": 0.32205305500002623, "nonpadding_tokens": 12127, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3271558679999771, "nonpadding_tokens": 25119, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3239250180000113, "nonpadding_tokens": 18402, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.32441379599998754, "nonpadding_tokens": 17859, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3272884799999929, "nonpadding_tokens": 10666, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.32600413899996283, "nonpadding_tokens": 13684, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3270385569999803, "nonpadding_tokens": 25064, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.32766882600003555, "nonpadding_tokens": 27556, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003759660000355325, "prefetch": {"data_wait_seconds": 0.0004329779999494576, "consumed_batches": 45, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 595, "tokens": 54346859, "loss_tokens": 54453, "loss": 2.7777745257377924, "lr": 0.00029996831148437585, "optimizer_lrs": [0.019997887432291726, 0.00029996831148437585], "gradient_norm": 0.49854403734207153, "seconds": 0.31640457700001434, "nonpadding_tokens_per_second": 172099.28034636975, "ranks": [{"compute_seconds": 0.31308750300001975, "nonpadding_tokens": 8985, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3118531169999983, "nonpadding_tokens": 5175, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31272014600000375, "nonpadding_tokens": 5481, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.313139872000022, "nonpadding_tokens": 12214, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31199086600003056, "nonpadding_tokens": 12071, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3141756530000066, "nonpadding_tokens": 3935, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3115146600000003, "nonpadding_tokens": 4289, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.31186317800001007, "nonpadding_tokens": 2303, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003438609999761866, "prefetch": {"data_wait_seconds": 0.0004417269999663631, "consumed_batches": 46, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 596, "tokens": 54440232, "loss_tokens": 93373, "loss": 2.9153204754050956, "lr": 0.00029996813896454886, "optimizer_lrs": [0.019997875930969924, 0.00029996813896454886], "gradient_norm": 0.37889087200164795, "seconds": 0.33438233600003286, "nonpadding_tokens_per_second": 279240.22876612365, "ranks": [{"compute_seconds": 0.3290176170000336, "nonpadding_tokens": 4967, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.33151695999998765, "nonpadding_tokens": 4806, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32872591300002796, "nonpadding_tokens": 1792, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3295165590000124, "nonpadding_tokens": 19530, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32763960199997655, "nonpadding_tokens": 19951, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.33020539999995435, "nonpadding_tokens": 7623, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3314048830000047, "nonpadding_tokens": 19176, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.33207978699999785, "nonpadding_tokens": 15528, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003474220000043715, "prefetch": {"data_wait_seconds": 0.00045098999994763744, "consumed_batches": 47, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 597, "tokens": 54531978, "loss_tokens": 91746, "loss": 2.9437427789767403, "lr": 0.0002999679689947132, "optimizer_lrs": [0.019997864599647548, 0.0002999679689947132], "gradient_norm": 0.5316601395606995, "seconds": 0.32073652299999367, "nonpadding_tokens_per_second": 286047.87238403095, "ranks": [{"compute_seconds": 0.31705993100001706, "nonpadding_tokens": 12399, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3155462270000271, "nonpadding_tokens": 15563, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31596051799999714, "nonpadding_tokens": 9355, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3183670509999956, "nonpadding_tokens": 5664, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31464820700000473, "nonpadding_tokens": 12494, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3173330129999954, "nonpadding_tokens": 6465, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3148029930000007, "nonpadding_tokens": 11428, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.31549743400000807, "nonpadding_tokens": 18378, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003895510000120339, "prefetch": {"data_wait_seconds": 0.0004609649999594012, "consumed_batches": 48, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 598, "tokens": 54615920, "loss_tokens": 83942, "loss": 2.8415529845012033, "lr": 0.00029996781308660764, "optimizer_lrs": [0.019997854205773845, 0.00029996781308660764], "gradient_norm": 0.4466627836227417, "seconds": 0.4061079659999791, "nonpadding_tokens_per_second": 206698.72799295033, "ranks": [{"compute_seconds": 0.40062208799997734, "nonpadding_tokens": 8380, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.40354135899997345, "nonpadding_tokens": 3691, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.40020883800002593, "nonpadding_tokens": 14562, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.40049549200000456, "nonpadding_tokens": 12202, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3988926220000053, "nonpadding_tokens": 15973, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.4018355990000373, "nonpadding_tokens": 12771, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.39906290299995817, "nonpadding_tokens": 12438, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.40361355900000717, "nonpadding_tokens": 3925, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003852920000326776, "prefetch": {"data_wait_seconds": 0.00046921099995245186, "consumed_batches": 49, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 599, "tokens": 54700442, "loss_tokens": 84522, "loss": 3.0310000650718156, "lr": 0.00029996765571884966, "optimizer_lrs": [0.01999784371458998, 0.00029996765571884966], "gradient_norm": 0.4162127375602722, "seconds": 0.39586030100002745, "nonpadding_tokens_per_second": 213514.7166474623, "ranks": [{"compute_seconds": 0.3920242250000001, "nonpadding_tokens": 20218, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3935346749999553, "nonpadding_tokens": 13179, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3921347170000331, "nonpadding_tokens": 1261, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.393451309999989, "nonpadding_tokens": 9608, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.38927018799995494, "nonpadding_tokens": 8674, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3916330189999826, "nonpadding_tokens": 16482, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.38888620399995943, "nonpadding_tokens": 13278, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3901079260000415, "nonpadding_tokens": 1822, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003406370000220704, "prefetch": {"data_wait_seconds": 0.0004780469999587922, "consumed_batches": 50, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 600, "tokens": 54801687, "loss_tokens": 101245, "loss": 3.448320287421601, "lr": 0.00029996746671017425, "optimizer_lrs": [0.019997831114011617, 0.00029996746671017425], "gradient_norm": 0.5394076704978943, "seconds": 0.28740257800001245, "nonpadding_tokens_per_second": 352275.8936421079, "ranks": [{"compute_seconds": 0.2513131080000335, "nonpadding_tokens": 10727, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.26958679199998414, "nonpadding_tokens": 8558, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26664994600002956, "nonpadding_tokens": 10245, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.26737606199998254, "nonpadding_tokens": 12857, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26548130100002254, "nonpadding_tokens": 16646, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.26850814599998785, "nonpadding_tokens": 9575, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.26538495699998066, "nonpadding_tokens": 16331, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.27032461300001387, "nonpadding_tokens": 16306, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.007430069999998068, "prefetch": {"data_wait_seconds": 0.00048671599996907844, "consumed_batches": 51, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 601, "tokens": 54858904, "loss_tokens": 57217, "loss": 3.0183905133089817, "lr": 0.00029996735965143504, "optimizer_lrs": [0.01999782397676234, 0.00029996735965143504], "gradient_norm": 0.5092041492462158, "seconds": 0.3192058099999713, "nonpadding_tokens_per_second": 179247.99050495087, "ranks": [{"compute_seconds": 0.3160820789999548, "nonpadding_tokens": 8165, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3165306700000201, "nonpadding_tokens": 4125, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3135363810000058, "nonpadding_tokens": 13729, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.31382912200001556, "nonpadding_tokens": 2894, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31654321199999913, "nonpadding_tokens": 2724, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3154743720000397, "nonpadding_tokens": 8100, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31614221799998177, "nonpadding_tokens": 8584, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3168167530000119, "nonpadding_tokens": 8896, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039702399999441695, "prefetch": {"data_wait_seconds": 0.0004962649999384894, "consumed_batches": 52, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 602, "tokens": 54969432, "loss_tokens": 110528, "loss": 3.1900660918500288, "lr": 0.0002999671523445515, "optimizer_lrs": [0.019997810156303437, 0.0002999671523445515], "gradient_norm": 0.4217408001422882, "seconds": 0.4145212850000348, "nonpadding_tokens_per_second": 266640.10751580755, "ranks": [{"compute_seconds": 0.41174195499996813, "nonpadding_tokens": 17982, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.4121164260000114, "nonpadding_tokens": 13481, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41043565799998305, "nonpadding_tokens": 14924, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.4094854759999862, "nonpadding_tokens": 7519, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4075918870000237, "nonpadding_tokens": 25853, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.4110151620000124, "nonpadding_tokens": 11082, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.4078053329999989, "nonpadding_tokens": 17292, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.40767795299996124, "nonpadding_tokens": 2395, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035306400002355076, "prefetch": {"data_wait_seconds": 0.0005060979999740312, "consumed_batches": 53, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 603, "tokens": 55070042, "loss_tokens": 100610, "loss": 3.021767530563562, "lr": 0.0002999669630694314, "optimizer_lrs": [0.019997797537962096, 0.0002999669630694314], "gradient_norm": 0.4078725576400757, "seconds": 0.6098381940000195, "nonpadding_tokens_per_second": 164978.18764037068, "ranks": [{"compute_seconds": 0.6055014130000131, "nonpadding_tokens": 13652, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.6036853379999911, "nonpadding_tokens": 10196, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.6044418639999662, "nonpadding_tokens": 14316, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.6068400000000338, "nonpadding_tokens": 11563, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.6073683750000214, "nonpadding_tokens": 15412, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.6061491780000097, "nonpadding_tokens": 13951, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.607347543000003, "nonpadding_tokens": 13997, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.6041764579999835, "nonpadding_tokens": 7523, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00048160199997937525, "prefetch": {"data_wait_seconds": 0.0005159839999606675, "consumed_batches": 54, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 604, "tokens": 55129978, "loss_tokens": 59936, "loss": 2.8166950997730913, "lr": 0.00029996685005488795, "optimizer_lrs": [0.0199977900036592, 0.00029996685005488795], "gradient_norm": 0.47648683190345764, "seconds": 0.2065220410000279, "nonpadding_tokens_per_second": 290215.99684845214, "ranks": [{"compute_seconds": 0.20147016099997472, "nonpadding_tokens": 1274, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.19979819500002804, "nonpadding_tokens": 5840, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20076801699997304, "nonpadding_tokens": 6990, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.20115244900000562, "nonpadding_tokens": 5722, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.20004946900002096, "nonpadding_tokens": 7826, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.20235923499996034, "nonpadding_tokens": 4257, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.19973712000000887, "nonpadding_tokens": 5945, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.2042352309999842, "nonpadding_tokens": 22082, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034065400001281887, "prefetch": {"data_wait_seconds": 0.0005244049999788558, "consumed_batches": 55, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 605, "tokens": 55233726, "loss_tokens": 103748, "loss": 3.2581676273277558, "lr": 0.00029996665397294387, "optimizer_lrs": [0.01999777693152959, 0.00029996665397294387], "gradient_norm": 0.46140506863594055, "seconds": 0.2879297170000541, "nonpadding_tokens_per_second": 360324.0439401415, "ranks": [{"compute_seconds": 0.2833545390000154, "nonpadding_tokens": 8979, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.28505700800002387, "nonpadding_tokens": 4697, "peak_allocated_bytes": 14980623872, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28206379699997797, "nonpadding_tokens": 10457, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.28248554900000045, "nonpadding_tokens": 21586, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28101543700000775, "nonpadding_tokens": 15539, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.2838718619999554, "nonpadding_tokens": 1647, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.28062006699997255, "nonpadding_tokens": 28936, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.2856600669999807, "nonpadding_tokens": 11907, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034272699997472955, "prefetch": {"data_wait_seconds": 0.0005321889999692075, "consumed_batches": 56, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 606, "tokens": 55317089, "loss_tokens": 83363, "loss": 3.3354530787039813, "lr": 0.00029996649599940093, "optimizer_lrs": [0.019997766399960062, 0.00029996649599940093], "gradient_norm": 0.43082988262176514, "seconds": 0.3062347059999979, "nonpadding_tokens_per_second": 272219.30880688806, "ranks": [{"compute_seconds": 0.3024789549999696, "nonpadding_tokens": 12593, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.300723435000009, "nonpadding_tokens": 12989, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3033094189999588, "nonpadding_tokens": 6840, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3012970440000231, "nonpadding_tokens": 3317, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3003878259999624, "nonpadding_tokens": 11742, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.30278602499998897, "nonpadding_tokens": 9283, "peak_allocated_bytes": 14977003008, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30397486000003937, "nonpadding_tokens": 14418, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.30059644900001103, "nonpadding_tokens": 12181, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003364860000374392, "prefetch": {"data_wait_seconds": 0.0005403209999741421, "consumed_batches": 57, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 607, "tokens": 55423441, "loss_tokens": 106352, "loss": 3.03577124783737, "lr": 0.0002999662939196651, "optimizer_lrs": [0.019997752927977674, 0.0002999662939196651], "gradient_norm": 0.40871748328208923, "seconds": 0.3242814870000075, "nonpadding_tokens_per_second": 327961.9844595, "ranks": [{"compute_seconds": 0.3203448019999655, "nonpadding_tokens": 8013, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3218037340000137, "nonpadding_tokens": 3311, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3186574300000302, "nonpadding_tokens": 12669, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3190446149999957, "nonpadding_tokens": 14916, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3216028160000519, "nonpadding_tokens": 6631, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3200222080000117, "nonpadding_tokens": 18988, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3214055149999808, "nonpadding_tokens": 23015, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3219384189999914, "nonpadding_tokens": 18809, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003864939999971284, "prefetch": {"data_wait_seconds": 0.0005487419999923304, "consumed_batches": 58, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 608, "tokens": 55558627, "loss_tokens": 135186, "loss": 2.9525743697572233, "lr": 0.0002999660361754573, "optimizer_lrs": [0.019997735745030488, 0.0002999660361754573], "gradient_norm": 0.5668042302131653, "seconds": 0.3687806820000219, "nonpadding_tokens_per_second": 366575.6006167155, "ranks": [{"compute_seconds": 0.3642871739999691, "nonpadding_tokens": 21166, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3662805510000453, "nonpadding_tokens": 20184, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36544939400005205, "nonpadding_tokens": 11415, "peak_allocated_bytes": 14987296256, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3664608189999967, "nonpadding_tokens": 23239, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3621256399999879, "nonpadding_tokens": 21538, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3650302259999876, "nonpadding_tokens": 9121, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3659844759999942, "nonpadding_tokens": 17757, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3629195800000389, "nonpadding_tokens": 10766, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003327060000515303, "prefetch": {"data_wait_seconds": 0.0005569059999999126, "consumed_batches": 59, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 609, "tokens": 55652320, "loss_tokens": 93693, "loss": 2.938919529740749, "lr": 0.00029996585696546535, "optimizer_lrs": [0.01999772379769769, 0.00029996585696546535], "gradient_norm": 0.5067319273948669, "seconds": 0.30581557699997575, "nonpadding_tokens_per_second": 306370.9210600722, "ranks": [{"compute_seconds": 0.30272546499998043, "nonpadding_tokens": 2239, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.302212542999996, "nonpadding_tokens": 19606, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30217610399995465, "nonpadding_tokens": 16139, "peak_allocated_bytes": 14988202496, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.29973408300003257, "nonpadding_tokens": 17948, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2984376879999786, "nonpadding_tokens": 16222, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.30098443299999644, "nonpadding_tokens": 11497, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3029529169999705, "nonpadding_tokens": 3676, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.30345211299999164, "nonpadding_tokens": 6366, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003632489999745303, "prefetch": {"data_wait_seconds": 0.000565979999990418, "consumed_batches": 60, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 610, "tokens": 55719166, "loss_tokens": 66846, "loss": 2.8773930938276036, "lr": 0.00029996572881850394, "optimizer_lrs": [0.01999771525456693, 0.00029996572881850394], "gradient_norm": 0.7674465775489807, "seconds": 0.37159415999997236, "nonpadding_tokens_per_second": 179889.80235858652, "ranks": [{"compute_seconds": 0.36620263399998976, "nonpadding_tokens": 11586, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3687597669999718, "nonpadding_tokens": 11401, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.367769423000027, "nonpadding_tokens": 3428, "peak_allocated_bytes": 14988202496, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3657636270000353, "nonpadding_tokens": 3768, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3640502119999951, "nonpadding_tokens": 4966, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3675443320000227, "nonpadding_tokens": 10758, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3642923529999962, "nonpadding_tokens": 6060, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.369062448999955, "nonpadding_tokens": 14879, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040542800002185686, "prefetch": {"data_wait_seconds": 0.0005757299999942234, "consumed_batches": 61, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 611, "tokens": 55813423, "loss_tokens": 94257, "loss": 3.0176883414494413, "lr": 0.00029996554771554747, "optimizer_lrs": [0.0199977031810365, 0.00029996554771554747], "gradient_norm": 0.4140896201133728, "seconds": 0.39694185500002277, "nonpadding_tokens_per_second": 237457.95212247042, "ranks": [{"compute_seconds": 0.3922270020000269, "nonpadding_tokens": 11413, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.39459058699998195, "nonpadding_tokens": 13497, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39180411199998844, "nonpadding_tokens": 6754, "peak_allocated_bytes": 14988202496, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3920315510000023, "nonpadding_tokens": 19684, "peak_allocated_bytes": 14987719168, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3944098530000133, "nonpadding_tokens": 7367, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3914462370000251, "nonpadding_tokens": 18716, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3940950649999877, "nonpadding_tokens": 7225, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3946228429999792, "nonpadding_tokens": 9601, "peak_allocated_bytes": 14982787584, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003768170000171267, "prefetch": {"data_wait_seconds": 0.0005845729999833793, "consumed_batches": 62, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 612, "tokens": 55894166, "loss_tokens": 80743, "loss": 3.021676027643263, "lr": 0.0002999653921985707, "optimizer_lrs": [0.019997692813238047, 0.0002999653921985707], "gradient_norm": 0.436374694108963, "seconds": 0.44675584300000537, "nonpadding_tokens_per_second": 180731.82760812604, "ranks": [{"compute_seconds": 0.42497469000005594, "nonpadding_tokens": 12540, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.43725551999995105, "nonpadding_tokens": 4171, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.43659620500000074, "nonpadding_tokens": 8537, "peak_allocated_bytes": 14988202496, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.43474429400004055, "nonpadding_tokens": 14860, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4373070890000008, "nonpadding_tokens": 8715, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.43566528399998106, "nonpadding_tokens": 4867, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.4369240759999684, "nonpadding_tokens": 10108, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.4341992150000351, "nonpadding_tokens": 16945, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.007252484000048298, "prefetch": {"data_wait_seconds": 0.0005924579999856405, "consumed_batches": 63, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 613, "tokens": 55971877, "loss_tokens": 77711, "loss": 2.862111702333003, "lr": 0.0002999652421907611, "optimizer_lrs": [0.01999768281271741, 0.0002999652421907611], "gradient_norm": 0.9720596075057983, "seconds": 0.28111223600001267, "nonpadding_tokens_per_second": 276441.1862882998, "ranks": [{"compute_seconds": 0.27883542900002567, "nonpadding_tokens": 8650, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.27360014999999294, "nonpadding_tokens": 9411, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27558771500002877, "nonpadding_tokens": 2929, "peak_allocated_bytes": 14988202496, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.27732885099999294, "nonpadding_tokens": 5489, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2748434389999943, "nonpadding_tokens": 5814, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.2776339619999817, "nonpadding_tokens": 5310, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2745445049999944, "nonpadding_tokens": 16370, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.27560301999994863, "nonpadding_tokens": 23738, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038805700000921206, "prefetch": {"data_wait_seconds": 0.0006011030000223627, "consumed_batches": 64, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 614, "tokens": 56085961, "loss_tokens": 114084, "loss": 3.172677588443603, "lr": 0.00029996502138347113, "optimizer_lrs": [0.01999766809223141, 0.00029996502138347113], "gradient_norm": 0.40555980801582336, "seconds": 0.3212603450000415, "nonpadding_tokens_per_second": 355113.8563335143, "ranks": [{"compute_seconds": 0.3180428819999861, "nonpadding_tokens": 10305, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.315091074999998, "nonpadding_tokens": 16311, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3158954439999775, "nonpadding_tokens": 10654, "peak_allocated_bytes": 14988202496, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.31819153499998265, "nonpadding_tokens": 7955, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3189378369999645, "nonpadding_tokens": 19718, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.31776723400002993, "nonpadding_tokens": 14404, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3147004109999898, "nonpadding_tokens": 17037, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3153409189999934, "nonpadding_tokens": 17700, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003506190000166498, "prefetch": {"data_wait_seconds": 0.0006099220000379546, "consumed_batches": 65, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 615, "tokens": 56201795, "loss_tokens": 115834, "loss": 3.063569957870746, "lr": 0.0002999647964738685, "optimizer_lrs": [0.019997653098257902, 0.0002999647964738685], "gradient_norm": 0.5663865208625793, "seconds": 0.4049542140000426, "nonpadding_tokens_per_second": 286042.2141451967, "ranks": [{"compute_seconds": 0.40022012400004314, "nonpadding_tokens": 15720, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.4025008220000359, "nonpadding_tokens": 10709, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4014958369999704, "nonpadding_tokens": 25450, "peak_allocated_bytes": 14988202496, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.40045580600002495, "nonpadding_tokens": 10526, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.40263291399998025, "nonpadding_tokens": 13666, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.40137708299999986, "nonpadding_tokens": 13579, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.40248152999998865, "nonpadding_tokens": 16821, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.39876857600000903, "nonpadding_tokens": 9363, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003758619999985058, "prefetch": {"data_wait_seconds": 0.0006191750000681395, "consumed_batches": 66, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 616, "tokens": 56344146, "loss_tokens": 142351, "loss": 2.700980103757613, "lr": 0.00029996451909040125, "optimizer_lrs": [0.019997634606026753, 0.00029996451909040125], "gradient_norm": 0.4399981200695038, "seconds": 0.32496219900002643, "nonpadding_tokens_per_second": 438054.027323924, "ranks": [{"compute_seconds": 0.31990615300003356, "nonpadding_tokens": 4018, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3221734480000009, "nonpadding_tokens": 13303, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31856013399999483, "nonpadding_tokens": 3690, "peak_allocated_bytes": 14988202496, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3193018649999999, "nonpadding_tokens": 22537, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3219998589999591, "nonpadding_tokens": 32032, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3208012079999776, "nonpadding_tokens": 21808, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31829720199999656, "nonpadding_tokens": 18114, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3225908740000136, "nonpadding_tokens": 26849, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039178699995545685, "prefetch": {"data_wait_seconds": 0.000627710000074444, "consumed_batches": 67, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 617, "tokens": 56429665, "loss_tokens": 85519, "loss": 3.108686373788281, "lr": 0.000299964351925758, "optimizer_lrs": [0.019997623461717202, 0.000299964351925758], "gradient_norm": 0.4642711281776428, "seconds": 0.3614666130000046, "nonpadding_tokens_per_second": 236588.93221211253, "ranks": [{"compute_seconds": 0.3566250650000029, "nonpadding_tokens": 12232, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.35841821400003937, "nonpadding_tokens": 11955, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35792689699997027, "nonpadding_tokens": 16379, "peak_allocated_bytes": 14988202496, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3563946280000323, "nonpadding_tokens": 9514, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35520785300002444, "nonpadding_tokens": 4898, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3574403139999731, "nonpadding_tokens": 4304, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.35886305100001437, "nonpadding_tokens": 1292, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.35577942100002247, "nonpadding_tokens": 24945, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003284609999809618, "prefetch": {"data_wait_seconds": 0.0006361980000662015, "consumed_batches": 68, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 618, "tokens": 56521701, "loss_tokens": 92036, "loss": 2.7153909204007127, "lr": 0.00029996417158343025, "optimizer_lrs": [0.019997611438895353, 0.00029996417158343025], "gradient_norm": 0.5538753271102905, "seconds": 0.3352534910000031, "nonpadding_tokens_per_second": 274526.5969504823, "ranks": [{"compute_seconds": 0.3288643050000246, "nonpadding_tokens": 10269, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.33211582799998496, "nonpadding_tokens": 2203, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33087635099997215, "nonpadding_tokens": 3984, "peak_allocated_bytes": 14988202496, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3290315259999943, "nonpadding_tokens": 24253, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32825177500001246, "nonpadding_tokens": 13978, "peak_allocated_bytes": 14973719040, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3292493509999872, "nonpadding_tokens": 17568, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.33160163600001624, "nonpadding_tokens": 11803, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3328592839999942, "nonpadding_tokens": 7978, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003969729999653282, "prefetch": {"data_wait_seconds": 0.0006458870000898287, "consumed_batches": 69, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 619, "tokens": 56641833, "loss_tokens": 120132, "loss": 2.8016811715446344, "lr": 0.00029996393550317497, "optimizer_lrs": [0.019997595700211666, 0.00029996393550317497], "gradient_norm": 0.3825317919254303, "seconds": 0.3130534770000395, "nonpadding_tokens_per_second": 383742.7430968443, "ranks": [{"compute_seconds": 0.3094814600000291, "nonpadding_tokens": 10105, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.310804419999954, "nonpadding_tokens": 14776, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3089969989999872, "nonpadding_tokens": 30428, "peak_allocated_bytes": 14988202496, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3082335220000232, "nonpadding_tokens": 13583, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3100017119999734, "nonpadding_tokens": 17916, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.31042756400000826, "nonpadding_tokens": 12517, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30674222900000814, "nonpadding_tokens": 6771, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3075707639999905, "nonpadding_tokens": 14036, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036495500000910397, "prefetch": {"data_wait_seconds": 0.0006561370000923716, "consumed_batches": 70, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 620, "tokens": 56730198, "loss_tokens": 88365, "loss": 2.8494702370848186, "lr": 0.0002999637613558185, "optimizer_lrs": [0.0199975840903879, 0.0002999637613558185], "gradient_norm": 0.4207383394241333, "seconds": 0.3516159470000275, "nonpadding_tokens_per_second": 251311.12725098638, "ranks": [{"compute_seconds": 0.347622605999959, "nonpadding_tokens": 13457, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.345276374999969, "nonpadding_tokens": 15501, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34638690700001007, "nonpadding_tokens": 12788, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3485297869999613, "nonpadding_tokens": 10556, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34492072499995174, "nonpadding_tokens": 6423, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.34657004999996843, "nonpadding_tokens": 6996, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3486751199999958, "nonpadding_tokens": 16895, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.34932194399999617, "nonpadding_tokens": 5749, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003828989999874466, "prefetch": {"data_wait_seconds": 0.0006678030000699664, "consumed_batches": 71, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 621, "tokens": 56820636, "loss_tokens": 90438, "loss": 3.286873327583538, "lr": 0.00029996358268878534, "optimizer_lrs": [0.019997572179252358, 0.00029996358268878534], "gradient_norm": 0.6682695150375366, "seconds": 0.35905246000004354, "nonpadding_tokens_per_second": 251879.62784042486, "ranks": [{"compute_seconds": 0.3543977030000178, "nonpadding_tokens": 10855, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3567329269999959, "nonpadding_tokens": 18489, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3552358009999921, "nonpadding_tokens": 7950, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3555088780000233, "nonpadding_tokens": 1797, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35289660199998707, "nonpadding_tokens": 9066, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3558810539999513, "nonpadding_tokens": 22163, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3527190329999712, "nonpadding_tokens": 10719, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.35664853899999116, "nonpadding_tokens": 9399, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003751069999680112, "prefetch": {"data_wait_seconds": 0.0006770080000393364, "consumed_batches": 72, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 622, "tokens": 56923250, "loss_tokens": 102614, "loss": 2.7782837868127155, "lr": 0.0002999633794351573, "optimizer_lrs": [0.019997558629010488, 0.0002999633794351573], "gradient_norm": 0.4924333691596985, "seconds": 0.38602440199997545, "nonpadding_tokens_per_second": 265822.5735688246, "ranks": [{"compute_seconds": 0.3833669020000343, "nonpadding_tokens": 25661, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.38370297099999107, "nonpadding_tokens": 10501, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37982358599998634, "nonpadding_tokens": 5811, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.380222257000014, "nonpadding_tokens": 9688, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3788881369999899, "nonpadding_tokens": 10036, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.38171468900003447, "nonpadding_tokens": 4298, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.379007109999975, "nonpadding_tokens": 23115, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3797882449999861, "nonpadding_tokens": 13504, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003904669999883481, "prefetch": {"data_wait_seconds": 0.0006860730000113335, "consumed_batches": 73, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 623, "tokens": 57002478, "loss_tokens": 79228, "loss": 3.0832088245317313, "lr": 0.00029996322211666365, "optimizer_lrs": [0.01999754814111091, 0.00029996322211666365], "gradient_norm": 0.44048234820365906, "seconds": 0.349343685000008, "nonpadding_tokens_per_second": 226790.98950936578, "ranks": [{"compute_seconds": 0.346962671999961, "nonpadding_tokens": 10295, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.34676339699996106, "nonpadding_tokens": 10050, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3433019810000246, "nonpadding_tokens": 17015, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.34500430800000004, "nonpadding_tokens": 11268, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34658481400003893, "nonpadding_tokens": 3959, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.34521920499997805, "nonpadding_tokens": 17039, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.34646694700001035, "nonpadding_tokens": 6759, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3427223809999873, "nonpadding_tokens": 2843, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003893129999710254, "prefetch": {"data_wait_seconds": 0.0006954100000058361, "consumed_batches": 74, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 624, "tokens": 57083532, "loss_tokens": 81054, "loss": 2.53433629123794, "lr": 0.0002999630608234988, "optimizer_lrs": [0.019997537388233254, 0.0002999630608234988], "gradient_norm": 0.4234408140182495, "seconds": 0.3529922440000064, "nonpadding_tokens_per_second": 229619.77600844548, "ranks": [{"compute_seconds": 0.33589796699999397, "nonpadding_tokens": 10069, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3397164350000139, "nonpadding_tokens": 13571, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34018977300002007, "nonpadding_tokens": 17681, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.341342625999971, "nonpadding_tokens": 3459, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34341123200005086, "nonpadding_tokens": 10071, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3423256130000141, "nonpadding_tokens": 10992, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3391433130000223, "nonpadding_tokens": 12168, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3401579720000427, "nonpadding_tokens": 3043, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005406849999758379, "prefetch": {"data_wait_seconds": 0.000705564000000436, "consumed_batches": 75, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 625, "tokens": 57158750, "loss_tokens": 75218, "loss": 3.326829390903773, "lr": 0.0002999629108279979, "optimizer_lrs": [0.019997527388533196, 0.0002999629108279979], "gradient_norm": 0.44909974932670593, "seconds": 0.36238999399995464, "nonpadding_tokens_per_second": 207560.91847284674, "ranks": [{"compute_seconds": 0.3594178739999734, "nonpadding_tokens": 11430, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.35954206600001726, "nonpadding_tokens": 11108, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3583883910000054, "nonpadding_tokens": 5715, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3567600099999595, "nonpadding_tokens": 17231, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3555234250000012, "nonpadding_tokens": 8010, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18549309440}, {"compute_seconds": 0.3567015989999618, "nonpadding_tokens": 9667, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3589903609999965, "nonpadding_tokens": 9986, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3600379270000076, "nonpadding_tokens": 2071, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004309139999918443, "prefetch": {"data_wait_seconds": 0.000715008000000239, "consumed_batches": 76, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 626, "tokens": 57243851, "loss_tokens": 85101, "loss": 2.9512256025193593, "lr": 0.00029996274075798346, "optimizer_lrs": [0.01999751605053223, 0.00029996274075798346], "gradient_norm": 0.6510472893714905, "seconds": 0.3564879040000051, "nonpadding_tokens_per_second": 238720.58222766174, "ranks": [{"compute_seconds": 0.35404998199999227, "nonpadding_tokens": 25295, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3541762219999782, "nonpadding_tokens": 10524, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3511255150000352, "nonpadding_tokens": 1852, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.35358611099997006, "nonpadding_tokens": 10607, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35419392800002925, "nonpadding_tokens": 9163, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3511871550000478, "nonpadding_tokens": 8507, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3535636319999753, "nonpadding_tokens": 8779, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.35253829900000255, "nonpadding_tokens": 10374, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003748400000063157, "prefetch": {"data_wait_seconds": 0.000724420999986819, "consumed_batches": 77, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 627, "tokens": 57329477, "loss_tokens": 85626, "loss": 2.5228882363417653, "lr": 0.0002999625692461991, "optimizer_lrs": [0.019997504616413274, 0.0002999625692461991], "gradient_norm": 0.5067712068557739, "seconds": 0.39501246899999387, "nonpadding_tokens_per_second": 216767.8408146689, "ranks": [{"compute_seconds": 0.3900999619999652, "nonpadding_tokens": 21562, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.39211793300000863, "nonpadding_tokens": 11505, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39122115499998245, "nonpadding_tokens": 9709, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.39178263200000174, "nonpadding_tokens": 3148, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3921007930000542, "nonpadding_tokens": 18759, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3895947820000174, "nonpadding_tokens": 4146, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3922904469999935, "nonpadding_tokens": 4817, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.39265003299999535, "nonpadding_tokens": 11980, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035810199995012226, "prefetch": {"data_wait_seconds": 0.000732944999995766, "consumed_batches": 78, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 628, "tokens": 57413255, "loss_tokens": 83778, "loss": 2.9918305297930243, "lr": 0.0002999624010548871, "optimizer_lrs": [0.01999749340365914, 0.0002999624010548871], "gradient_norm": 0.4092217683792114, "seconds": 0.3982139259999826, "nonpadding_tokens_per_second": 210384.4052907473, "ranks": [{"compute_seconds": 0.38670850900001597, "nonpadding_tokens": 13071, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.39598882999996476, "nonpadding_tokens": 11477, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3950102270000002, "nonpadding_tokens": 12490, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.39379217699996616, "nonpadding_tokens": 11758, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.39191912199999024, "nonpadding_tokens": 18002, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3935415099999773, "nonpadding_tokens": 6211, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3916450730000065, "nonpadding_tokens": 6407, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.39272813600001655, "nonpadding_tokens": 4362, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035739300000159346, "prefetch": {"data_wait_seconds": 0.0007404969999953437, "consumed_batches": 79, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 629, "tokens": 57481329, "loss_tokens": 68074, "loss": 2.9069606237329966, "lr": 0.000299962264113059, "optimizer_lrs": [0.019997484274203933, 0.000299962264113059], "gradient_norm": 0.5952762365341187, "seconds": 0.26129690999999866, "nonpadding_tokens_per_second": 260523.55536849, "ranks": [{"compute_seconds": 0.25888535300003923, "nonpadding_tokens": 7024, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2561032730000079, "nonpadding_tokens": 9126, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2589728649999756, "nonpadding_tokens": 7640, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.25688045900000134, "nonpadding_tokens": 6839, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2556561729999771, "nonpadding_tokens": 11048, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.25864379300003293, "nonpadding_tokens": 9841, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2552402129999791, "nonpadding_tokens": 6686, "peak_allocated_bytes": 14985367040, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.25664773199997626, "nonpadding_tokens": 9870, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038271500000064407, "prefetch": {"data_wait_seconds": 0.0007505800000018326, "consumed_batches": 80, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 630, "tokens": 57546886, "loss_tokens": 65557, "loss": 2.77675653438992, "lr": 0.00029996213199932133, "optimizer_lrs": [0.019997475466621423, 0.00029996213199932133], "gradient_norm": 0.42593884468078613, "seconds": 0.3167014800000061, "nonpadding_tokens_per_second": 206999.34840847205, "ranks": [{"compute_seconds": 0.3112689729999829, "nonpadding_tokens": 9400, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3139282519999824, "nonpadding_tokens": 6250, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3104227559999799, "nonpadding_tokens": 7813, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.31323699900002566, "nonpadding_tokens": 3345, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31380332700001645, "nonpadding_tokens": 3514, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.31244969600004424, "nonpadding_tokens": 15936, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31383244800002785, "nonpadding_tokens": 12659, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3143589119999888, "nonpadding_tokens": 6640, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036290899998903114, "prefetch": {"data_wait_seconds": 0.0007592299999856778, "consumed_batches": 81, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 631, "tokens": 57642266, "loss_tokens": 95380, "loss": 3.0498394579576433, "lr": 0.0002999619393725593, "optimizer_lrs": [0.019997462624837287, 0.0002999619393725593], "gradient_norm": 0.47214025259017944, "seconds": 0.31517296900000247, "nonpadding_tokens_per_second": 302627.4756449664, "ranks": [{"compute_seconds": 0.31258226400001377, "nonpadding_tokens": 12496, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.310216008999987, "nonpadding_tokens": 7243, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.312895492999985, "nonpadding_tokens": 24091, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.31127043000003596, "nonpadding_tokens": 7692, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30954868700001725, "nonpadding_tokens": 8387, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.312564186999964, "nonpadding_tokens": 13285, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3097089539999729, "nonpadding_tokens": 10402, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3107588320000332, "nonpadding_tokens": 11784, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034876500001246313, "prefetch": {"data_wait_seconds": 0.000768099999973515, "consumed_batches": 82, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 632, "tokens": 57735419, "loss_tokens": 93153, "loss": 2.7620485585542065, "lr": 0.000299961750771748, "optimizer_lrs": [0.01999745005144987, 0.000299961750771748], "gradient_norm": 0.4026416540145874, "seconds": 0.4151087259999713, "nonpadding_tokens_per_second": 224406.2679617254, "ranks": [{"compute_seconds": 0.41069664499997316, "nonpadding_tokens": 13293, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.41023917899997286, "nonpadding_tokens": 19256, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41224109700004874, "nonpadding_tokens": 9996, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.4080865369999742, "nonpadding_tokens": 3710, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.40561173399999007, "nonpadding_tokens": 2595, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.4103296279999995, "nonpadding_tokens": 6557, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.4126302379999629, "nonpadding_tokens": 25636, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.40668497199999365, "nonpadding_tokens": 12110, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034764599996606194, "prefetch": {"data_wait_seconds": 0.0007768669999705935, "consumed_batches": 83, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 633, "tokens": 57808708, "loss_tokens": 73289, "loss": 3.044867962791142, "lr": 0.000299961602060699, "optimizer_lrs": [0.019997440137379936, 0.000299961602060699], "gradient_norm": 0.45607537031173706, "seconds": 0.3523943000000145, "nonpadding_tokens_per_second": 207974.4195635315, "ranks": [{"compute_seconds": 0.34923407100001214, "nonpadding_tokens": 5381, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3488345310000227, "nonpadding_tokens": 9588, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3484860910000407, "nonpadding_tokens": 7590, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.34993716200000335, "nonpadding_tokens": 10077, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34516223200000695, "nonpadding_tokens": 10901, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.34681951600003913, "nonpadding_tokens": 10419, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.34481859799996073, "nonpadding_tokens": 10039, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.34581774100001894, "nonpadding_tokens": 9294, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041689299996505724, "prefetch": {"data_wait_seconds": 0.0008036480000015445, "consumed_batches": 84, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 634, "tokens": 57897820, "loss_tokens": 89112, "loss": 2.800846476905467, "lr": 0.00029996142085448875, "optimizer_lrs": [0.01999742805696592, 0.00029996142085448875], "gradient_norm": 0.400913268327713, "seconds": 0.3536738949999858, "nonpadding_tokens_per_second": 251960.92010128024, "ranks": [{"compute_seconds": 0.35127696100005323, "nonpadding_tokens": 11907, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.34796047099996485, "nonpadding_tokens": 12491, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34804413099999465, "nonpadding_tokens": 5360, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.35063970999999583, "nonpadding_tokens": 10429, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34743855700003223, "nonpadding_tokens": 8880, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.35032245200000034, "nonpadding_tokens": 4584, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3512087670000028, "nonpadding_tokens": 14837, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.34826546399995095, "nonpadding_tokens": 20624, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041534799998999006, "prefetch": {"data_wait_seconds": 0.0008146010000160686, "consumed_batches": 85, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 635, "tokens": 57962946, "loss_tokens": 65126, "loss": 2.8578366359057825, "lr": 0.0002999612881532375, "optimizer_lrs": [0.019997419210215833, 0.0002999612881532375], "gradient_norm": 0.4370717406272888, "seconds": 0.34877802500000143, "nonpadding_tokens_per_second": 186726.21361394465, "ranks": [{"compute_seconds": 0.3418589639999823, "nonpadding_tokens": 7709, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3424151549999692, "nonpadding_tokens": 8923, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3397856029999957, "nonpadding_tokens": 13173, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3400570829999765, "nonpadding_tokens": 1101, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3383059399999979, "nonpadding_tokens": 15060, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3416131569999834, "nonpadding_tokens": 8527, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3382862779999982, "nonpadding_tokens": 2613, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3430223290000072, "nonpadding_tokens": 8020, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.002807087999997293, "prefetch": {"data_wait_seconds": 0.0008236790000211158, "consumed_batches": 86, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 636, "tokens": 58082511, "loss_tokens": 119565, "loss": 3.395861560657383, "lr": 0.0002999610439336298, "optimizer_lrs": [0.019997402928908656, 0.0002999610439336298], "gradient_norm": 0.3643856644630432, "seconds": 0.33852831699999797, "nonpadding_tokens_per_second": 353190.54269838444, "ranks": [{"compute_seconds": 0.3360956920000149, "nonpadding_tokens": 19216, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3330434640000135, "nonpadding_tokens": 6879, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33388671700004124, "nonpadding_tokens": 9118, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3342510470000093, "nonpadding_tokens": 12084, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3324181019999628, "nonpadding_tokens": 11030, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.33546904000002087, "nonpadding_tokens": 18681, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.336167042999989, "nonpadding_tokens": 16951, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.33329525900001045, "nonpadding_tokens": 25606, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003946790000100009, "prefetch": {"data_wait_seconds": 0.0008325980000591926, "consumed_batches": 87, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 637, "tokens": 58171331, "loss_tokens": 88820, "loss": 2.7389555421076333, "lr": 0.00029996086201572, "optimizer_lrs": [0.019997390801048, 0.00029996086201572], "gradient_norm": 0.42166489362716675, "seconds": 0.3882155590000025, "nonpadding_tokens_per_second": 228790.41795437012, "ranks": [{"compute_seconds": 0.3838657230000422, "nonpadding_tokens": 5808, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3818173650000176, "nonpadding_tokens": 11723, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.384616861999973, "nonpadding_tokens": 6890, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3829115209999827, "nonpadding_tokens": 21828, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3854743559999747, "nonpadding_tokens": 2843, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.38273913899996614, "nonpadding_tokens": 20479, "peak_allocated_bytes": 14979501568, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.38126824599999054, "nonpadding_tokens": 8822, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.385912114000007, "nonpadding_tokens": 10427, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003779479999934665, "prefetch": {"data_wait_seconds": 0.0008419340000642705, "consumed_batches": 88, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 638, "tokens": 58266725, "loss_tokens": 95394, "loss": 2.641939122219427, "lr": 0.0002999606661612675, "optimizer_lrs": [0.0199973777440845, 0.0002999606661612675], "gradient_norm": 0.5733497142791748, "seconds": 0.3422524989999829, "nonpadding_tokens_per_second": 278724.04227501276, "ranks": [{"compute_seconds": 0.3399370710000085, "nonpadding_tokens": 10632, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.33972771399999147, "nonpadding_tokens": 9340, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33848958800001583, "nonpadding_tokens": 8144, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.33678952799999706, "nonpadding_tokens": 4039, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3357826240000463, "nonpadding_tokens": 16962, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3370668169999931, "nonpadding_tokens": 19068, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.33568008099996405, "nonpadding_tokens": 7513, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.34000135399998044, "nonpadding_tokens": 19696, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003353619999870716, "prefetch": {"data_wait_seconds": 0.0008500470000996074, "consumed_batches": 89, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 639, "tokens": 58353812, "loss_tokens": 87087, "loss": 3.1838843627636733, "lr": 0.000299960486935241, "optimizer_lrs": [0.019997365795682735, 0.000299960486935241], "gradient_norm": 0.4531332850456238, "seconds": 0.31463825299999826, "nonpadding_tokens_per_second": 276784.52689603664, "ranks": [{"compute_seconds": 0.31111074099999314, "nonpadding_tokens": 9446, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3089953630000082, "nonpadding_tokens": 2667, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30986881900003027, "nonpadding_tokens": 8541, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.31204371900003025, "nonpadding_tokens": 9070, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3082997980000073, "nonpadding_tokens": 6127, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3095951099999752, "nonpadding_tokens": 20969, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31230152099999486, "nonpadding_tokens": 12276, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.30912982299997793, "nonpadding_tokens": 17991, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039068899997118933, "prefetch": {"data_wait_seconds": 0.0008596030001086774, "consumed_batches": 90, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 640, "tokens": 58417287, "loss_tokens": 63475, "loss": 2.5705952146514375, "lr": 0.0002999603560463296, "optimizer_lrs": [0.01999735706975531, 0.0002999603560463296], "gradient_norm": 0.3783201277256012, "seconds": 0.31517511300000933, "nonpadding_tokens_per_second": 201395.9776068935, "ranks": [{"compute_seconds": 0.2882816119999916, "nonpadding_tokens": 6794, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.28543795299998465, "nonpadding_tokens": 1514, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28633109300000115, "nonpadding_tokens": 11933, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.28703116899998804, "nonpadding_tokens": 18107, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2894628870000133, "nonpadding_tokens": 5461, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2863625659999798, "nonpadding_tokens": 2189, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2851914029999989, "nonpadding_tokens": 7953, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.2894929209999759, "nonpadding_tokens": 9524, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.012454887999979292, "prefetch": {"data_wait_seconds": 0.0008683040000647679, "consumed_batches": 91, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 641, "tokens": 58520679, "loss_tokens": 103392, "loss": 2.906247582017951, "lr": 0.0002999601423830744, "optimizer_lrs": [0.019997342825538295, 0.0002999601423830744], "gradient_norm": 0.34647634625434875, "seconds": 0.343107257999975, "nonpadding_tokens_per_second": 301340.1715914956, "ranks": [{"compute_seconds": 0.33896349700000883, "nonpadding_tokens": 13967, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.33732168700004195, "nonpadding_tokens": 11003, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33864334499997994, "nonpadding_tokens": 13224, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.33793525199996566, "nonpadding_tokens": 8418, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3408108680000055, "nonpadding_tokens": 17279, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3395621159999678, "nonpadding_tokens": 11367, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3406561769999712, "nonpadding_tokens": 12334, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.33740109199999324, "nonpadding_tokens": 15800, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003764580000051865, "prefetch": {"data_wait_seconds": 0.0008782650000540571, "consumed_batches": 92, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 642, "tokens": 58611933, "loss_tokens": 91254, "loss": 3.064757435290508, "lr": 0.0002999599533264338, "optimizer_lrs": [0.01999733022176225, 0.0002999599533264338], "gradient_norm": 0.43533286452293396, "seconds": 0.4280395879999901, "nonpadding_tokens_per_second": 213190.5612431393, "ranks": [{"compute_seconds": 0.4251457479999772, "nonpadding_tokens": 16891, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.42224702699996897, "nonpadding_tokens": 11339, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.42490942000000587, "nonpadding_tokens": 23519, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.42306536500001357, "nonpadding_tokens": 7912, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4220715939999877, "nonpadding_tokens": 14808, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.42283778099999836, "nonpadding_tokens": 9479, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.42178159999997433, "nonpadding_tokens": 376, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.42217893800000184, "nonpadding_tokens": 6930, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003975280000076964, "prefetch": {"data_wait_seconds": 0.0008882980000208818, "consumed_batches": 93, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 643, "tokens": 58699938, "loss_tokens": 88005, "loss": 2.939156333446963, "lr": 0.0002999597705773033, "optimizer_lrs": [0.019997318038486887, 0.0002999597705773033], "gradient_norm": 0.5348037481307983, "seconds": 0.3174166580000133, "nonpadding_tokens_per_second": 277253.87997751625, "ranks": [{"compute_seconds": 0.31392505100001245, "nonpadding_tokens": 9409, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3150940199999468, "nonpadding_tokens": 16670, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3118798950000041, "nonpadding_tokens": 13990, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3124530489999984, "nonpadding_tokens": 12189, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31125982700001487, "nonpadding_tokens": 17391, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3138709119999703, "nonpadding_tokens": 6809, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31467175200003794, "nonpadding_tokens": 8858, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.31192440099999885, "nonpadding_tokens": 2689, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003977990000407772, "prefetch": {"data_wait_seconds": 0.0008969140000232301, "consumed_batches": 94, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 644, "tokens": 58785204, "loss_tokens": 85266, "loss": 3.0438554054371028, "lr": 0.0002999595931191661, "optimizer_lrs": [0.019997306207944408, 0.0002999595931191661], "gradient_norm": 0.45594024658203125, "seconds": 0.3153881590000083, "nonpadding_tokens_per_second": 270352.572114154, "ranks": [{"compute_seconds": 0.3121942639999702, "nonpadding_tokens": 4300, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3092317709999861, "nonpadding_tokens": 6105, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30997361799995815, "nonpadding_tokens": 10220, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.30991457200002515, "nonpadding_tokens": 20255, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30864939899998944, "nonpadding_tokens": 4931, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3102809760000014, "nonpadding_tokens": 12640, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30906237599998576, "nonpadding_tokens": 11907, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3131189950000248, "nonpadding_tokens": 14908, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003694679999739492, "prefetch": {"data_wait_seconds": 0.0009062959999965869, "consumed_batches": 95, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 645, "tokens": 58890619, "loss_tokens": 105415, "loss": 2.773304321016933, "lr": 0.0002999593731865406, "optimizer_lrs": [0.019997291545769375, 0.0002999593731865406], "gradient_norm": 0.39693689346313477, "seconds": 0.3715471579999985, "nonpadding_tokens_per_second": 283719.03197278775, "ranks": [{"compute_seconds": 0.3684591300000193, "nonpadding_tokens": 9377, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.36581204899999875, "nonpadding_tokens": 10217, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.368319041999996, "nonpadding_tokens": 17330, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.36853650199998356, "nonpadding_tokens": 13290, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3692473110000378, "nonpadding_tokens": 13810, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.36799565200004736, "nonpadding_tokens": 15557, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3648311450000392, "nonpadding_tokens": 18623, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.36590533399999003, "nonpadding_tokens": 7211, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036516000000119675, "prefetch": {"data_wait_seconds": 0.0009152259999751777, "consumed_batches": 96, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 646, "tokens": 58994254, "loss_tokens": 103635, "loss": 2.87592693105611, "lr": 0.00029995915638582284, "optimizer_lrs": [0.01999727709238819, 0.00029995915638582284], "gradient_norm": 0.40697646141052246, "seconds": 0.30975124700000833, "nonpadding_tokens_per_second": 334574.9242455744, "ranks": [{"compute_seconds": 0.3075777000000244, "nonpadding_tokens": 9803, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.30749120799998764, "nonpadding_tokens": 1576, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30627041399998234, "nonpadding_tokens": 9625, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.30484519899999896, "nonpadding_tokens": 874, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30369222899997794, "nonpadding_tokens": 18142, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.30726002800003016, "nonpadding_tokens": 24480, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30333052399998905, "nonpadding_tokens": 22526, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3042278730000021, "nonpadding_tokens": 16609, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003524929999798587, "prefetch": {"data_wait_seconds": 0.0009250679999723843, "consumed_batches": 97, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 647, "tokens": 59069546, "loss_tokens": 75292, "loss": 2.754541070100409, "lr": 0.00029995899851587384, "optimizer_lrs": [0.019997266567724926, 0.00029995899851587384], "gradient_norm": 0.39129209518432617, "seconds": 1.4581728500000395, "nonpadding_tokens_per_second": 51634.482153468954, "ranks": [{"compute_seconds": 1.4552570500000002, "nonpadding_tokens": 14565, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 1.4558772810000278, "nonpadding_tokens": 15193, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 1.4529359079999722, "nonpadding_tokens": 4384, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 1.4539667220000183, "nonpadding_tokens": 12150, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 1.4519161019999842, "nonpadding_tokens": 12120, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 1.4547886309999853, "nonpadding_tokens": 6512, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 1.4516404569999963, "nonpadding_tokens": 4993, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 1.4526927420000106, "nonpadding_tokens": 5375, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000342389000024923, "prefetch": {"data_wait_seconds": 0.000933397999986596, "consumed_batches": 98, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 648, "tokens": 59158980, "loss_tokens": 89434, "loss": 2.7726078029608425, "lr": 0.0002999588105977948, "optimizer_lrs": [0.019997254039852988, 0.0002999588105977948], "gradient_norm": 0.4258131980895996, "seconds": 0.3211800199999857, "nonpadding_tokens_per_second": 278454.43187905644, "ranks": [{"compute_seconds": 0.31825177899997925, "nonpadding_tokens": 5393, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.31787766700000475, "nonpadding_tokens": 17782, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31506498700002794, "nonpadding_tokens": 18266, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3151524790000053, "nonpadding_tokens": 12932, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3136732250000023, "nonpadding_tokens": 19441, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3157917220000286, "nonpadding_tokens": 3518, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31373758799998086, "nonpadding_tokens": 5703, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3188332950000472, "nonpadding_tokens": 6399, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038387999995848077, "prefetch": {"data_wait_seconds": 0.0009423610000141025, "consumed_batches": 99, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 649, "tokens": 59278479, "loss_tokens": 119499, "loss": 2.6962759625603563, "lr": 0.00029995855883687713, "optimizer_lrs": [0.01999723725579181, 0.00029995855883687713], "gradient_norm": 0.436198353767395, "seconds": 0.32882969899998216, "nonpadding_tokens_per_second": 363406.9561338694, "ranks": [{"compute_seconds": 0.32357916699999123, "nonpadding_tokens": 12034, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3262232499999982, "nonpadding_tokens": 12250, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3229761729999723, "nonpadding_tokens": 15985, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3255461140000193, "nonpadding_tokens": 23348, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3221242639999673, "nonpadding_tokens": 14918, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3234049009999467, "nonpadding_tokens": 9642, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3219095729999708, "nonpadding_tokens": 12975, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.32650374100001045, "nonpadding_tokens": 18347, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003893400000265501, "prefetch": {"data_wait_seconds": 0.0009512110000429175, "consumed_batches": 100, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 650, "tokens": 59357103, "loss_tokens": 78624, "loss": 2.9800041176994303, "lr": 0.00029995839277324136, "optimizer_lrs": [0.01999722618488276, 0.00029995839277324136], "gradient_norm": 0.3976914882659912, "seconds": 0.36381882000000587, "nonpadding_tokens_per_second": 216107.56694774266, "ranks": [{"compute_seconds": 0.3598706889999903, "nonpadding_tokens": 8771, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.35876413799996953, "nonpadding_tokens": 18869, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36126583899999787, "nonpadding_tokens": 13408, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3612840459999802, "nonpadding_tokens": 10557, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3578467729999488, "nonpadding_tokens": 6865, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3613448639999888, "nonpadding_tokens": 9088, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3579609450000021, "nonpadding_tokens": 9952, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.35899141800001644, "nonpadding_tokens": 1114, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003909829999884096, "prefetch": {"data_wait_seconds": 0.0009608320000467074, "consumed_batches": 101, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 651, "tokens": 59434096, "loss_tokens": 76993, "loss": 2.980815626095879, "lr": 0.00029995822983272925, "optimizer_lrs": [0.01999721532218195, 0.00029995822983272925], "gradient_norm": 0.5900915265083313, "seconds": 0.35665781300002664, "nonpadding_tokens_per_second": 215873.5830076832, "ranks": [{"compute_seconds": 0.3535158360000423, "nonpadding_tokens": 13060, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3542853179999952, "nonpadding_tokens": 12174, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35063996100001305, "nonpadding_tokens": 6770, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3530164050000053, "nonpadding_tokens": 11097, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34946939200000315, "nonpadding_tokens": 4257, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.35283210099998996, "nonpadding_tokens": 3927, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3497906879999846, "nonpadding_tokens": 17880, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3504134980000231, "nonpadding_tokens": 7828, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000416729999983545, "prefetch": {"data_wait_seconds": 0.0009688399999987496, "consumed_batches": 102, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 652, "tokens": 59522119, "loss_tokens": 88023, "loss": 2.907254531486089, "lr": 0.00029995804315933474, "optimizer_lrs": [0.019997202877288983, 0.00029995804315933474], "gradient_norm": 0.45436689257621765, "seconds": 0.3346467089999692, "nonpadding_tokens_per_second": 263032.618079648, "ranks": [{"compute_seconds": 0.33195793799995954, "nonpadding_tokens": 8380, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.32786785600001167, "nonpadding_tokens": 16211, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33030285500001355, "nonpadding_tokens": 9593, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.33066478000000643, "nonpadding_tokens": 8330, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3314707179999914, "nonpadding_tokens": 17884, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3323392040000499, "nonpadding_tokens": 3742, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.331479311999999, "nonpadding_tokens": 8022, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.33016167300002053, "nonpadding_tokens": 15861, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003743800000393094, "prefetch": {"data_wait_seconds": 0.0009777049999826204, "consumed_batches": 103, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 653, "tokens": 59624032, "loss_tokens": 101913, "loss": 2.8474519933668914, "lr": 0.00029995782650912845, "optimizer_lrs": [0.0199971884339419, 0.00029995782650912845], "gradient_norm": 0.47336092591285706, "seconds": 0.32488859599999387, "nonpadding_tokens_per_second": 313685.9873037893, "ranks": [{"compute_seconds": 0.32061630700002297, "nonpadding_tokens": 12956, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.31947618399999556, "nonpadding_tokens": 13586, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3221756840000012, "nonpadding_tokens": 11107, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.31976994199999353, "nonpadding_tokens": 13677, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3225804929999754, "nonpadding_tokens": 20980, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.31995683000002373, "nonpadding_tokens": 11671, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3184057290000055, "nonpadding_tokens": 7702, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3194957559999807, "nonpadding_tokens": 10234, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034792500002822635, "prefetch": {"data_wait_seconds": 0.00098655600000086, "consumed_batches": 104, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 654, "tokens": 59695679, "loss_tokens": 71647, "loss": 3.1252787102041957, "lr": 0.00029995767386550837, "optimizer_lrs": [0.01999717825770056, 0.00029995767386550837], "gradient_norm": 0.5060079097747803, "seconds": 0.3811180179999951, "nonpadding_tokens_per_second": 187991.63675331906, "ranks": [{"compute_seconds": 0.3787407149999922, "nonpadding_tokens": 7261, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.37883083400004125, "nonpadding_tokens": 7747, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37591940499999055, "nonpadding_tokens": 13627, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3769018389999701, "nonpadding_tokens": 6463, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.37472444299999097, "nonpadding_tokens": 3142, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3778016200000138, "nonpadding_tokens": 4583, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3745349069999975, "nonpadding_tokens": 9697, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3756272050000007, "nonpadding_tokens": 19127, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035773800004790246, "prefetch": {"data_wait_seconds": 0.0009955979999887177, "consumed_batches": 105, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 655, "tokens": 59814227, "loss_tokens": 118548, "loss": 3.6421378365725277, "lr": 0.0002999574206940301, "optimizer_lrs": [0.01999716137960201, 0.0002999574206940301], "gradient_norm": 0.43591931462287903, "seconds": 0.36416540099997974, "nonpadding_tokens_per_second": 325533.39684240514, "ranks": [{"compute_seconds": 0.36173650800003543, "nonpadding_tokens": 24390, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.35772373699995796, "nonpadding_tokens": 7746, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3607492000000434, "nonpadding_tokens": 21975, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.36135750299996516, "nonpadding_tokens": 13570, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3572194949999812, "nonpadding_tokens": 2855, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3602936080000063, "nonpadding_tokens": 17016, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3572724100000073, "nonpadding_tokens": 17260, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3582742780000103, "nonpadding_tokens": 13736, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040405900000450856, "prefetch": {"data_wait_seconds": 0.0010043919999844775, "consumed_batches": 106, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 656, "tokens": 59904437, "loss_tokens": 90210, "loss": 2.978905470568673, "lr": 0.0002999572275355581, "optimizer_lrs": [0.01999714850237054, 0.0002999572275355581], "gradient_norm": 0.47466298937797546, "seconds": 0.27358912400001145, "nonpadding_tokens_per_second": 329728.01945151965, "ranks": [{"compute_seconds": 0.26882047399999465, "nonpadding_tokens": 7607, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2709393309999655, "nonpadding_tokens": 17510, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26785181099995725, "nonpadding_tokens": 12342, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.26832205900001327, "nonpadding_tokens": 6361, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2665400859999636, "nonpadding_tokens": 11248, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2692873060000238, "nonpadding_tokens": 16167, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.26667311599999266, "nonpadding_tokens": 7196, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.2675698000000466, "nonpadding_tokens": 11779, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004265379999992547, "prefetch": {"data_wait_seconds": 0.0010139219999842908, "consumed_batches": 107, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 657, "tokens": 59990125, "loss_tokens": 85688, "loss": 2.8983853485902342, "lr": 0.00029995704365488244, "optimizer_lrs": [0.01999713624365883, 0.00029995704365488244], "gradient_norm": 0.45087340474128723, "seconds": 0.31264300300000514, "nonpadding_tokens_per_second": 274076.18010884634, "ranks": [{"compute_seconds": 0.308295309000016, "nonpadding_tokens": 12594, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3103190039999504, "nonpadding_tokens": 9225, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3071329740000124, "nonpadding_tokens": 5598, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3076313759999607, "nonpadding_tokens": 9790, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3060533169999644, "nonpadding_tokens": 11612, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.30750671999999213, "nonpadding_tokens": 11528, "peak_allocated_bytes": 14982129152, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3063133869999888, "nonpadding_tokens": 13185, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.30675941599997714, "nonpadding_tokens": 12156, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041987700001300254, "prefetch": {"data_wait_seconds": 0.0010240019999656624, "consumed_batches": 108, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 658, "tokens": 60094075, "loss_tokens": 103950, "loss": 3.124413780663781, "lr": 0.0002999568200558185, "optimizer_lrs": [0.019997121337054568, 0.0002999568200558185], "gradient_norm": 0.5543584227561951, "seconds": 0.3862895670000057, "nonpadding_tokens_per_second": 269098.6474402982, "ranks": [{"compute_seconds": 0.38302477799999224, "nonpadding_tokens": 18597, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3840444549999802, "nonpadding_tokens": 2897, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3825496009999938, "nonpadding_tokens": 10243, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3837206780000315, "nonpadding_tokens": 7996, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3794331340000099, "nonpadding_tokens": 4052, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3822459180000237, "nonpadding_tokens": 18525, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3793743140000174, "nonpadding_tokens": 23525, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.380145347999985, "nonpadding_tokens": 18115, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003363769999964461, "prefetch": {"data_wait_seconds": 0.0010317559999322157, "consumed_batches": 109, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 659, "tokens": 60181089, "loss_tokens": 87014, "loss": 2.6796948623210057, "lr": 0.00029995663244029896, "optimizer_lrs": [0.019997108829353266, 0.00029995663244029896], "gradient_norm": 0.5621213316917419, "seconds": 0.3842652849999695, "nonpadding_tokens_per_second": 226442.52134305317, "ranks": [{"compute_seconds": 0.31855269499999395, "nonpadding_tokens": 16576, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3736273990000427, "nonpadding_tokens": 10147, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37417251599998735, "nonpadding_tokens": 10590, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.37447807400002375, "nonpadding_tokens": 9769, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.37739793800000143, "nonpadding_tokens": 10455, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3758889930000464, "nonpadding_tokens": 16451, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3729714400000148, "nonpadding_tokens": 3180, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3733709020000333, "nonpadding_tokens": 9846, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.004897452999955476, "prefetch": {"data_wait_seconds": 0.0010398589999027763, "consumed_batches": 110, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 660, "tokens": 60250503, "loss_tokens": 69414, "loss": 3.3023795632005073, "lr": 0.00029995648248149056, "optimizer_lrs": [0.019997098832099373, 0.00029995648248149056], "gradient_norm": 0.566270649433136, "seconds": 0.3829129310000212, "nonpadding_tokens_per_second": 181278.8087848518, "ranks": [{"compute_seconds": 0.3795843139999988, "nonpadding_tokens": 9907, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.38058800999999676, "nonpadding_tokens": 4826, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3786003060000098, "nonpadding_tokens": 6010, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3789970139999923, "nonpadding_tokens": 6597, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3763967570000091, "nonpadding_tokens": 16288, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3791375899999707, "nonpadding_tokens": 12969, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3764406709999548, "nonpadding_tokens": 4791, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3800802920000024, "nonpadding_tokens": 8026, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035587100001066574, "prefetch": {"data_wait_seconds": 0.0010482529999222834, "consumed_batches": 111, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 661, "tokens": 60320410, "loss_tokens": 69907, "loss": 2.665144182628349, "lr": 0.00029995633119608535, "optimizer_lrs": [0.01999708874640569, 0.00029995633119608535], "gradient_norm": 0.44974878430366516, "seconds": 0.31050649000002295, "nonpadding_tokens_per_second": 225138.6114344819, "ranks": [{"compute_seconds": 0.3080384350000145, "nonpadding_tokens": 12843, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3055773560000148, "nonpadding_tokens": 10597, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3024347040000066, "nonpadding_tokens": 5764, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3026187710000272, "nonpadding_tokens": 578, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30109489299996994, "nonpadding_tokens": 9784, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.30480770799999846, "nonpadding_tokens": 4681, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30093034999998736, "nonpadding_tokens": 10521, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.30169284499999094, "nonpadding_tokens": 15139, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035284300003013414, "prefetch": {"data_wait_seconds": 0.0010562569999166271, "consumed_batches": 112, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 662, "tokens": 60403906, "loss_tokens": 83496, "loss": 3.041029660103478, "lr": 0.00029995615015883327, "optimizer_lrs": [0.019997076677255553, 0.00029995615015883327], "gradient_norm": 0.43844249844551086, "seconds": 0.3543086099999755, "nonpadding_tokens_per_second": 235658.96408784922, "ranks": [{"compute_seconds": 0.3518827209999813, "nonpadding_tokens": 11036, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.35154664500004174, "nonpadding_tokens": 14850, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34853535199999897, "nonpadding_tokens": 4591, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3491139009999529, "nonpadding_tokens": 17347, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.351886487999991, "nonpadding_tokens": 14428, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.34894227500001307, "nonpadding_tokens": 6009, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3472284800000125, "nonpadding_tokens": 9776, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.34798670300000367, "nonpadding_tokens": 5459, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003662620000000061, "prefetch": {"data_wait_seconds": 0.001064874999954668, "consumed_batches": 113, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 663, "tokens": 60512834, "loss_tokens": 108928, "loss": 3.094274428980611, "lr": 0.000299955913416683, "optimizer_lrs": [0.019997060894445536, 0.000299955913416683], "gradient_norm": 0.47600945830345154, "seconds": 0.35639277399997127, "nonpadding_tokens_per_second": 305640.31581630436, "ranks": [{"compute_seconds": 0.35367571799997677, "nonpadding_tokens": 11119, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3541387799999711, "nonpadding_tokens": 23270, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3511769589999858, "nonpadding_tokens": 16993, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3540228269999943, "nonpadding_tokens": 5473, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3539771370000153, "nonpadding_tokens": 6153, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3516383360000077, "nonpadding_tokens": 14580, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.35045694699999785, "nonpadding_tokens": 18397, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.35093787599998905, "nonpadding_tokens": 12943, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003898279999816623, "prefetch": {"data_wait_seconds": 0.0010736879999200255, "consumed_batches": 114, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 664, "tokens": 60609091, "loss_tokens": 96257, "loss": 3.237261965363558, "lr": 0.00029995570368308324, "optimizer_lrs": [0.01999704691220555, 0.00029995570368308324], "gradient_norm": 0.5128342509269714, "seconds": 0.37678054000002703, "nonpadding_tokens_per_second": 255472.32349099847, "ranks": [{"compute_seconds": 0.3716361950000078, "nonpadding_tokens": 13919, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.37045113299996046, "nonpadding_tokens": 6775, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3729207860000088, "nonpadding_tokens": 17797, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3716363789999946, "nonpadding_tokens": 12432, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3697602569999958, "nonpadding_tokens": 13197, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3735684559999868, "nonpadding_tokens": 9488, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3697344259999795, "nonpadding_tokens": 10816, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.37441676299999926, "nonpadding_tokens": 11833, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003809590000400931, "prefetch": {"data_wait_seconds": 0.0010833469999198542, "consumed_batches": 115, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 665, "tokens": 60718325, "loss_tokens": 109234, "loss": 2.6580148465679185, "lr": 0.0002999554650712311, "optimizer_lrs": [0.01999703100474874, 0.0002999554650712311], "gradient_norm": 0.4317343533039093, "seconds": 0.38726515300004394, "nonpadding_tokens_per_second": 282065.14103784494, "ranks": [{"compute_seconds": 0.3821514750000006, "nonpadding_tokens": 20487, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3810590040000079, "nonpadding_tokens": 17238, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3810377140000014, "nonpadding_tokens": 8750, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3816273330000399, "nonpadding_tokens": 11047, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3799483349999946, "nonpadding_tokens": 11695, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3826895980000131, "nonpadding_tokens": 18425, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.38404142100000627, "nonpadding_tokens": 4224, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3848808989999952, "nonpadding_tokens": 17368, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034497500001862136, "prefetch": {"data_wait_seconds": 0.0010927469999302275, "consumed_batches": 116, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 666, "tokens": 60822264, "loss_tokens": 103939, "loss": 2.5913474489845005, "lr": 0.0002999552374308369, "optimizer_lrs": [0.019997015828722462, 0.0002999552374308369], "gradient_norm": 0.3600154519081116, "seconds": 0.3804861329999767, "nonpadding_tokens_per_second": 273174.2131585289, "ranks": [{"compute_seconds": 0.3678349619999608, "nonpadding_tokens": 21609, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3725079480000204, "nonpadding_tokens": 13931, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37480025000002115, "nonpadding_tokens": 11333, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3757619679999493, "nonpadding_tokens": 16260, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.37192567899995765, "nonpadding_tokens": 3056, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3746748210000419, "nonpadding_tokens": 14662, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3757047630000443, "nonpadding_tokens": 12235, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.37265076800002817, "nonpadding_tokens": 10853, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038373699999283417, "prefetch": {"data_wait_seconds": 0.0011160869999145007, "consumed_batches": 117, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 667, "tokens": 60952499, "loss_tokens": 130235, "loss": 3.097747821246209, "lr": 0.00029995495137969633, "optimizer_lrs": [0.019996996758646423, 0.00029995495137969633], "gradient_norm": 0.5406211018562317, "seconds": 0.3344509180000159, "nonpadding_tokens_per_second": 389399.43947169493, "ranks": [{"compute_seconds": 0.32928694900004984, "nonpadding_tokens": 11929, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3313261920000059, "nonpadding_tokens": 15024, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32841472600000543, "nonpadding_tokens": 13975, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.33073175999999194, "nonpadding_tokens": 20633, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.331803501999957, "nonpadding_tokens": 29092, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3286538670000141, "nonpadding_tokens": 26937, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.33128174699999136, "nonpadding_tokens": 6816, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.3321316860000252, "nonpadding_tokens": 5829, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037050100002034014, "prefetch": {"data_wait_seconds": 0.0011240189999313088, "consumed_batches": 118, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 668, "tokens": 61066809, "loss_tokens": 114310, "loss": 2.8792554238474324, "lr": 0.00029995469955593153, "optimizer_lrs": [0.019996979970395436, 0.00029995469955593153], "gradient_norm": 0.6098107695579529, "seconds": 0.3432139720000009, "nonpadding_tokens_per_second": 333057.53648047784, "ranks": [{"compute_seconds": 0.34099684900002103, "nonpadding_tokens": 9495, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.33658195499998556, "nonpadding_tokens": 6650, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33764866899997514, "nonpadding_tokens": 10960, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3397497980000139, "nonpadding_tokens": 13681, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34038847499999747, "nonpadding_tokens": 11208, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3391795430000002, "nonpadding_tokens": 21816, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3363858520000349, "nonpadding_tokens": 17402, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.33664167500000985, "nonpadding_tokens": 23098, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00032749899997952525, "prefetch": {"data_wait_seconds": 0.0011321799999706172, "consumed_batches": 119, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 669, "tokens": 61199558, "loss_tokens": 132749, "loss": 3.1628677522994524, "lr": 0.0002999544062305317, "optimizer_lrs": [0.019996960415368782, 0.0002999544062305317], "gradient_norm": 0.8154848217964172, "seconds": 0.405010591000007, "nonpadding_tokens_per_second": 327766.737339463, "ranks": [{"compute_seconds": 0.40100995299997066, "nonpadding_tokens": 25703, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.39845227800003613, "nonpadding_tokens": 18663, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4008985479999865, "nonpadding_tokens": 12425, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.399690784000029, "nonpadding_tokens": 4788, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3982714020000344, "nonpadding_tokens": 14066, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.39882215800002996, "nonpadding_tokens": 16905, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3977469349999865, "nonpadding_tokens": 27845, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.4026793129999646, "nonpadding_tokens": 12354, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003784280000331819, "prefetch": {"data_wait_seconds": 0.001140004999982125, "consumed_batches": 120, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 670, "tokens": 61272381, "loss_tokens": 72823, "loss": 2.8076234500089257, "lr": 0.00029995424491706586, "optimizer_lrs": [0.019996949661137724, 0.00029995424491706586], "gradient_norm": 0.4227885901927948, "seconds": 0.5223367890000077, "nonpadding_tokens_per_second": 139417.71196973632, "ranks": [{"compute_seconds": 0.519219631999988, "nonpadding_tokens": 13158, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.5200660550000293, "nonpadding_tokens": 2739, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5165191449999611, "nonpadding_tokens": 12644, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.5170490250000057, "nonpadding_tokens": 15077, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.5196035730000403, "nonpadding_tokens": 5725, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.5186019979999514, "nonpadding_tokens": 8067, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.5192704620000086, "nonpadding_tokens": 3691, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17626562560}, {"compute_seconds": 0.5163240650000489, "nonpadding_tokens": 11722, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034600099996850986, "prefetch": {"data_wait_seconds": 0.0011492519999620754, "consumed_batches": 121, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 671, "tokens": 61344488, "loss_tokens": 72107, "loss": 2.9348208218342187, "lr": 0.00029995408490901396, "optimizer_lrs": [0.019996938993934267, 0.00029995408490901396], "gradient_norm": 0.45151233673095703, "seconds": 0.29317190700004403, "nonpadding_tokens_per_second": 245954.67122976817, "ranks": [{"compute_seconds": 0.2901416429999699, "nonpadding_tokens": 21043, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2909348860000023, "nonpadding_tokens": 6724, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2898959839999975, "nonpadding_tokens": 5846, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.2884372790000498, "nonpadding_tokens": 13051, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28666359700002886, "nonpadding_tokens": 6796, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2894769150000229, "nonpadding_tokens": 8696, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.29055202500001087, "nonpadding_tokens": 3666, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.28743357199999764, "nonpadding_tokens": 6285, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036966499999380176, "prefetch": {"data_wait_seconds": 0.001158025999998813, "consumed_batches": 122, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 672, "tokens": 61433813, "loss_tokens": 89325, "loss": 2.628010775258886, "lr": 0.000299953886306373, "optimizer_lrs": [0.019996925753758204, 0.000299953886306373], "gradient_norm": 0.38411641120910645, "seconds": 0.32729715299996087, "nonpadding_tokens_per_second": 272917.1310573871, "ranks": [{"compute_seconds": 0.3224495940000338, "nonpadding_tokens": 12428, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3205034009999963, "nonpadding_tokens": 13585, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3212884019999933, "nonpadding_tokens": 15363, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.32381266099997674, "nonpadding_tokens": 13385, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32483691799995995, "nonpadding_tokens": 5918, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3231261370000311, "nonpadding_tokens": 4225, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3204860829999916, "nonpadding_tokens": 14459, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.32150490300000456, "nonpadding_tokens": 9962, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000340418000007503, "prefetch": {"data_wait_seconds": 0.0011659730000275204, "consumed_batches": 123, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 673, "tokens": 61514620, "loss_tokens": 80807, "loss": 2.6714873092677616, "lr": 0.00029995370627323444, "optimizer_lrs": [0.019996913751548964, 0.00029995370627323444], "gradient_norm": 0.4666806757450104, "seconds": 0.36586904500001083, "nonpadding_tokens_per_second": 220863.1779712263, "ranks": [{"compute_seconds": 0.3634341249999693, "nonpadding_tokens": 14696, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.36067874700000857, "nonpadding_tokens": 17402, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.363502523999955, "nonpadding_tokens": 9679, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3619421319999674, "nonpadding_tokens": 1750, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3608219439999516, "nonpadding_tokens": 11580, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.36272871800002804, "nonpadding_tokens": 10691, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3599937000000182, "nonpadding_tokens": 11057, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3609792369999809, "nonpadding_tokens": 3952, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037908199999492354, "prefetch": {"data_wait_seconds": 0.001174598999980958, "consumed_batches": 124, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 674, "tokens": 61619708, "loss_tokens": 105088, "loss": 2.6342137066077953, "lr": 0.0002999534716189015, "optimizer_lrs": [0.019996898107926767, 0.0002999534716189015], "gradient_norm": 0.45277222990989685, "seconds": 0.32624508399999286, "nonpadding_tokens_per_second": 322113.66593343765, "ranks": [{"compute_seconds": 0.3231595940000034, "nonpadding_tokens": 8684, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3196259139999711, "nonpadding_tokens": 27029, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31752672800001847, "nonpadding_tokens": 10984, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.32274953000001005, "nonpadding_tokens": 13350, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3165305819999844, "nonpadding_tokens": 14502, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.320612734000008, "nonpadding_tokens": 12297, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31632001100001617, "nonpadding_tokens": 7310, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.32367330900001434, "nonpadding_tokens": 10932, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005260340000177166, "prefetch": {"data_wait_seconds": 0.0011833749999397014, "consumed_batches": 125, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 675, "tokens": 61732441, "loss_tokens": 112733, "loss": 2.6255122723603557, "lr": 0.00029995321923440975, "optimizer_lrs": [0.019996881282293984, 0.00029995321923440975], "gradient_norm": 0.595795750617981, "seconds": 0.31337808200004247, "nonpadding_tokens_per_second": 359734.79472627805, "ranks": [{"compute_seconds": 0.3071872170000347, "nonpadding_tokens": 9968, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.30736880899996777, "nonpadding_tokens": 12077, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31003996699996605, "nonpadding_tokens": 23741, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.30722060599998713, "nonpadding_tokens": 18689, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3109970059999796, "nonpadding_tokens": 11182, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3085046840000132, "nonpadding_tokens": 2153, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30567028299998356, "nonpadding_tokens": 15773, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.31090776799999276, "nonpadding_tokens": 19150, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004081410000367214, "prefetch": {"data_wait_seconds": 0.001192611999911719, "consumed_batches": 126, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 676, "tokens": 61877569, "loss_tokens": 145128, "loss": 2.764433164172317, "lr": 0.0002999528933196904, "optimizer_lrs": [0.019996859554646026, 0.0002999528933196904], "gradient_norm": 0.5206246376037598, "seconds": 0.462551440000027, "nonpadding_tokens_per_second": 313755.37388877553, "ranks": [{"compute_seconds": 0.45990244999995866, "nonpadding_tokens": 17098, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.4598286120000239, "nonpadding_tokens": 25017, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4587254690000009, "nonpadding_tokens": 26239, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.4572507459999997, "nonpadding_tokens": 17756, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4556881479999788, "nonpadding_tokens": 20337, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.45745910400000867, "nonpadding_tokens": 8713, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.45611523500002704, "nonpadding_tokens": 19315, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.46019363099998145, "nonpadding_tokens": 10653, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003464049999593044, "prefetch": {"data_wait_seconds": 0.0012010649999325551, "consumed_batches": 127, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 677, "tokens": 61967029, "loss_tokens": 89460, "loss": 3.038343603286385, "lr": 0.0002999526918553496, "optimizer_lrs": [0.019996846123689977, 0.0002999526918553496], "gradient_norm": 0.6169239282608032, "seconds": 0.35240500399999064, "nonpadding_tokens_per_second": 253855.6461587656, "ranks": [{"compute_seconds": 0.3498749769999563, "nonpadding_tokens": 12463, "peak_allocated_bytes": 15004352000, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.34662044899999955, "nonpadding_tokens": 3341, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3493010010000148, "nonpadding_tokens": 20469, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3498115670000175, "nonpadding_tokens": 7550, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35010027300000957, "nonpadding_tokens": 18632, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.34915519399999084, "nonpadding_tokens": 3322, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.34978177499999674, "nonpadding_tokens": 11018, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.34702853200002437, "nonpadding_tokens": 12665, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037613699998928496, "prefetch": {"data_wait_seconds": 0.001210192999920423, "consumed_batches": 128, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 678, "tokens": 62062816, "loss_tokens": 95787, "loss": 2.92134436301377, "lr": 0.00029995247566608874, "optimizer_lrs": [0.019996831711072586, 0.00029995247566608874], "gradient_norm": 0.41171184182167053, "seconds": 0.3044290979999573, "nonpadding_tokens_per_second": 314644.69273569057, "ranks": [{"compute_seconds": 0.30004921899995907, "nonpadding_tokens": 17676, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2985412740000015, "nonpadding_tokens": 6379, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3012707290000094, "nonpadding_tokens": 9844, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3015427820000127, "nonpadding_tokens": 13927, "peak_allocated_bytes": 14993407488, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2984339970000178, "nonpadding_tokens": 16034, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2998278329999948, "nonpadding_tokens": 6544, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3021232149999946, "nonpadding_tokens": 17568, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.299319464000007, "nonpadding_tokens": 7815, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000380012999983137, "prefetch": {"data_wait_seconds": 0.0012348529999286484, "consumed_batches": 129, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 679, "tokens": 62133704, "loss_tokens": 70888, "loss": 3.025375416149419, "lr": 0.000299952315356077, "optimizer_lrs": [0.019996821023738467, 0.000299952315356077], "gradient_norm": 0.4118759036064148, "seconds": 0.2965490199999863, "nonpadding_tokens_per_second": 239043.10997218362, "ranks": [{"compute_seconds": 0.29161160299997846, "nonpadding_tokens": 12493, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2938530349999837, "nonpadding_tokens": 5814, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2924723519999475, "nonpadding_tokens": 3203, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.2927235739999787, "nonpadding_tokens": 11605, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2893648520000056, "nonpadding_tokens": 4479, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.29072642199997745, "nonpadding_tokens": 18300, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2896694499999626, "nonpadding_tokens": 4477, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2942189780000035, "nonpadding_tokens": 10517, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000349290999963614, "prefetch": {"data_wait_seconds": 0.0012434649999164549, "consumed_batches": 130, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 680, "tokens": 62232840, "loss_tokens": 99136, "loss": 2.685956035143641, "lr": 0.00029995209071186936, "optimizer_lrs": [0.01999680604745796, 0.00029995209071186936], "gradient_norm": 0.4104337692260742, "seconds": 0.3117249949999632, "nonpadding_tokens_per_second": 318023.90437126067, "ranks": [{"compute_seconds": 0.3062645310000107, "nonpadding_tokens": 14194, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3053609760000313, "nonpadding_tokens": 8479, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3061773289999792, "nonpadding_tokens": 8196, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3067078519999882, "nonpadding_tokens": 16863, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30904577600000493, "nonpadding_tokens": 7617, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3061058260000209, "nonpadding_tokens": 8521, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3084970620000149, "nonpadding_tokens": 13965, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3091499000000226, "nonpadding_tokens": 21301, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004018509999923481, "prefetch": {"data_wait_seconds": 0.0012530039999347764, "consumed_batches": 131, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 681, "tokens": 62351407, "loss_tokens": 118567, "loss": 2.9819629934973473, "lr": 0.0002999518213434809, "optimizer_lrs": [0.01999678808956539, 0.0002999518213434809], "gradient_norm": 0.37668412923812866, "seconds": 0.38977355999998053, "nonpadding_tokens_per_second": 304194.56876450503, "ranks": [{"compute_seconds": 0.3850509559999864, "nonpadding_tokens": 18111, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3837663830000224, "nonpadding_tokens": 12848, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3843224689999829, "nonpadding_tokens": 25351, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.386786466999979, "nonpadding_tokens": 8267, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.38731909099999484, "nonpadding_tokens": 17072, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.38637745500000165, "nonpadding_tokens": 18120, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3833837459999927, "nonpadding_tokens": 8073, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.38390382000000045, "nonpadding_tokens": 10725, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003621509999902628, "prefetch": {"data_wait_seconds": 0.0012609539999317576, "consumed_batches": 132, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 682, "tokens": 62431354, "loss_tokens": 79947, "loss": 2.8489667373384866, "lr": 0.000299951639288404, "optimizer_lrs": [0.019996775952560268, 0.000299951639288404], "gradient_norm": 0.42222681641578674, "seconds": 0.3564921029999937, "nonpadding_tokens_per_second": 224260.22716133329, "ranks": [{"compute_seconds": 0.35325502000000597, "nonpadding_tokens": 11143, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3541100379999875, "nonpadding_tokens": 7097, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3512288339999827, "nonpadding_tokens": 5179, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3518272179999826, "nonpadding_tokens": 11248, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35415747500002226, "nonpadding_tokens": 7212, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3514229650000402, "nonpadding_tokens": 6117, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3538433370000007, "nonpadding_tokens": 12200, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3510793430000376, "nonpadding_tokens": 19751, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035003900001129296, "prefetch": {"data_wait_seconds": 0.0012698029999143046, "consumed_batches": 133, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 683, "tokens": 62510944, "loss_tokens": 79590, "loss": 2.4819863911923608, "lr": 0.000299951457705322, "optimizer_lrs": [0.019996763847021468, 0.000299951457705322], "gradient_norm": 0.5173708200454712, "seconds": 0.3185336860000234, "nonpadding_tokens_per_second": 249863.68317727675, "ranks": [{"compute_seconds": 0.31484431499995935, "nonpadding_tokens": 9255, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3158464619999677, "nonpadding_tokens": 5334, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3150935410000102, "nonpadding_tokens": 6468, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3140004810000505, "nonpadding_tokens": 13195, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.316230824999991, "nonpadding_tokens": 6744, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.31516759100003355, "nonpadding_tokens": 18982, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31225415799997336, "nonpadding_tokens": 11980, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3125105820000158, "nonpadding_tokens": 7632, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035942699997804084, "prefetch": {"data_wait_seconds": 0.0012800409999300655, "consumed_batches": 134, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 684, "tokens": 62575376, "loss_tokens": 64432, "loss": 2.624602536394959, "lr": 0.0002999513104557474, "optimizer_lrs": [0.01999675403038316, 0.0002999513104557474], "gradient_norm": 0.7054833769798279, "seconds": 0.3295582930000478, "nonpadding_tokens_per_second": 195510.17640448408, "ranks": [{"compute_seconds": 0.3244481279999718, "nonpadding_tokens": 7979, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3268850029999726, "nonpadding_tokens": 3011, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3237983319999671, "nonpadding_tokens": 9257, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3244456820000323, "nonpadding_tokens": 2435, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3231194569999616, "nonpadding_tokens": 3402, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.32527690800003484, "nonpadding_tokens": 12245, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.326352777000011, "nonpadding_tokens": 11714, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3272156560000212, "nonpadding_tokens": 14389, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038976900003717674, "prefetch": {"data_wait_seconds": 0.0012909039999300376, "consumed_batches": 135, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 685, "tokens": 62651509, "loss_tokens": 76133, "loss": 2.8937935914780715, "lr": 0.000299951136177942, "optimizer_lrs": [0.0199967424118628, 0.000299951136177942], "gradient_norm": 0.39018842577934265, "seconds": 0.27708048599998847, "nonpadding_tokens_per_second": 274768.53783201164, "ranks": [{"compute_seconds": 0.27232016299996076, "nonpadding_tokens": 6188, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.27442451400003165, "nonpadding_tokens": 6763, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2709672500000124, "nonpadding_tokens": 7912, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.2737626570000202, "nonpadding_tokens": 7051, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2701429159999975, "nonpadding_tokens": 9292, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27133198299998185, "nonpadding_tokens": 8794, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2700700089999941, "nonpadding_tokens": 5187, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2747322170000075, "nonpadding_tokens": 24946, "peak_allocated_bytes": 14988246528, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003836340000020755, "prefetch": {"data_wait_seconds": 0.0013002029999285014, "consumed_batches": 136, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 686, "tokens": 62758956, "loss_tokens": 107447, "loss": 2.8035912473126285, "lr": 0.00029995088968886517, "optimizer_lrs": [0.01999672597925768, 0.00029995088968886517], "gradient_norm": 1.2942068576812744, "seconds": 0.4315110080000295, "nonpadding_tokens_per_second": 249001.75895395153, "ranks": [{"compute_seconds": 0.427649869999982, "nonpadding_tokens": 16313, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.42885029699999677, "nonpadding_tokens": 13404, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4255987009999558, "nonpadding_tokens": 21088, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.4281198360000076, "nonpadding_tokens": 18502, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.42409671799998705, "nonpadding_tokens": 7806, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.42725663699997085, "nonpadding_tokens": 13389, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.4285567840000226, "nonpadding_tokens": 6550, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.4291136030000189, "nonpadding_tokens": 10395, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00032966700001679783, "prefetch": {"data_wait_seconds": 0.0013091459998832988, "consumed_batches": 137, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 687, "tokens": 62844270, "loss_tokens": 85314, "loss": 2.754015680310383, "lr": 0.00029995069353245873, "optimizer_lrs": [0.019996712902163916, 0.00029995069353245873], "gradient_norm": 0.6046563982963562, "seconds": 0.42796180499999537, "nonpadding_tokens_per_second": 199349.56578660314, "ranks": [{"compute_seconds": 0.42553335700000616, "nonpadding_tokens": 17086, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.42193178799999487, "nonpadding_tokens": 14629, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.42463228199994774, "nonpadding_tokens": 8747, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.42554043500001626, "nonpadding_tokens": 27374, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4215948930000195, "nonpadding_tokens": 4900, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.42447690000000193, "nonpadding_tokens": 2716, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.4212049140000431, "nonpadding_tokens": 6369, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.42226964700000735, "nonpadding_tokens": 3493, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037595899999587346, "prefetch": {"data_wait_seconds": 0.001320265999893877, "consumed_batches": 138, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 688, "tokens": 62930502, "loss_tokens": 86232, "loss": 2.432867511480657, "lr": 0.00029995049486813643, "optimizer_lrs": [0.019996699657875764, 0.00029995049486813643], "gradient_norm": 0.4202802777290344, "seconds": 0.350366245000032, "nonpadding_tokens_per_second": 246119.59979190383, "ranks": [{"compute_seconds": 0.3474256420000188, "nonpadding_tokens": 15623, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.34758735000002616, "nonpadding_tokens": 25634, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3465674869999589, "nonpadding_tokens": 9148, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3455939940000121, "nonpadding_tokens": 7511, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3442230310000127, "nonpadding_tokens": 11623, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3458521100000098, "nonpadding_tokens": 8047, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.34721557400001757, "nonpadding_tokens": 5305, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3479889849999722, "nonpadding_tokens": 3341, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035257799999044437, "prefetch": {"data_wait_seconds": 0.0013298669998818013, "consumed_batches": 139, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 689, "tokens": 63011285, "loss_tokens": 80783, "loss": 3.079007572756644, "lr": 0.00029995030839511863, "optimizer_lrs": [0.019996687226341243, 0.00029995030839511863], "gradient_norm": 0.4150804877281189, "seconds": 0.3291468930000292, "nonpadding_tokens_per_second": 245431.45239409214, "ranks": [{"compute_seconds": 0.32529446600000256, "nonpadding_tokens": 8850, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.32669574199996987, "nonpadding_tokens": 10661, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32367308900001035, "nonpadding_tokens": 17302, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.32624146099999507, "nonpadding_tokens": 9413, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3225743489999786, "nonpadding_tokens": 6656, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3254142829999864, "nonpadding_tokens": 8744, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.32649385600001324, "nonpadding_tokens": 12463, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.32323144400004367, "nonpadding_tokens": 6694, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004333399999723042, "prefetch": {"data_wait_seconds": 0.0013412199998725782, "consumed_batches": 140, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 690, "tokens": 63122258, "loss_tokens": 110973, "loss": 2.846278486658917, "lr": 0.00029995005166250446, "optimizer_lrs": [0.01999667011083363, 0.00029995005166250446], "gradient_norm": 0.3722611665725708, "seconds": 0.31061852300001647, "nonpadding_tokens_per_second": 357264.5923630064, "ranks": [{"compute_seconds": 0.3070145830000115, "nonpadding_tokens": 7030, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.30421257099999366, "nonpadding_tokens": 7957, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30718389899999465, "nonpadding_tokens": 9094, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3055246740000257, "nonpadding_tokens": 18266, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.308216042999959, "nonpadding_tokens": 19359, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.30678892299999916, "nonpadding_tokens": 6190, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30398856199997226, "nonpadding_tokens": 10309, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.304463887000054, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003932010000085029, "prefetch": {"data_wait_seconds": 0.0013502579998458941, "consumed_batches": 141, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 691, "tokens": 63262213, "loss_tokens": 139955, "loss": 2.944339028616341, "lr": 0.00029994972693790525, "optimizer_lrs": [0.01999664846252702, 0.00029994972693790525], "gradient_norm": 0.3629775047302246, "seconds": 0.32511269700000867, "nonpadding_tokens_per_second": 430481.49546738947, "ranks": [{"compute_seconds": 0.3196741730000099, "nonpadding_tokens": 9337, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3218878079999854, "nonpadding_tokens": 23441, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3191345959999694, "nonpadding_tokens": 11048, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.32007849799998667, "nonpadding_tokens": 22497, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32229647100001557, "nonpadding_tokens": 9773, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3208060619999742, "nonpadding_tokens": 25308, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3178511330000333, "nonpadding_tokens": 18436, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3226686190000123, "nonpadding_tokens": 20115, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003882029999999759, "prefetch": {"data_wait_seconds": 0.001358744999848227, "consumed_batches": 142, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 692, "tokens": 63333325, "loss_tokens": 71112, "loss": 2.80263176397795, "lr": 0.0002999495615402922, "optimizer_lrs": [0.019996637436019482, 0.0002999495615402922], "gradient_norm": 0.48319482803344727, "seconds": 0.28442621900001086, "nonpadding_tokens_per_second": 250019.14468369487, "ranks": [{"compute_seconds": 0.2820617599999764, "nonpadding_tokens": 6263, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2810477390000301, "nonpadding_tokens": 4752, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27781593700001395, "nonpadding_tokens": 12280, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.2780195450000065, "nonpadding_tokens": 11058, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2772855169999957, "nonpadding_tokens": 22645, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27894455900002413, "nonpadding_tokens": 4809, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2768290509999929, "nonpadding_tokens": 5490, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2775928609999596, "nonpadding_tokens": 3815, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035808400002679264, "prefetch": {"data_wait_seconds": 0.0013667099998571075, "consumed_batches": 143, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 693, "tokens": 63416280, "loss_tokens": 82955, "loss": 2.751885058164065, "lr": 0.00029994936825415307, "optimizer_lrs": [0.019996624550276874, 0.00029994936825415307], "gradient_norm": 0.4396771788597107, "seconds": 0.3386094399999706, "nonpadding_tokens_per_second": 244987.263202134, "ranks": [{"compute_seconds": 0.3354812959999549, "nonpadding_tokens": 13127, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3358742689999872, "nonpadding_tokens": 10880, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3330236880000257, "nonpadding_tokens": 5483, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.33380176200000733, "nonpadding_tokens": 10865, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33628898800003526, "nonpadding_tokens": 11044, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3335370160000366, "nonpadding_tokens": 6769, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.33160854099998005, "nonpadding_tokens": 13449, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3324529719999987, "nonpadding_tokens": 11338, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003594320000388507, "prefetch": {"data_wait_seconds": 0.001376799999846412, "consumed_batches": 144, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 694, "tokens": 63501282, "loss_tokens": 85002, "loss": 2.862451214383191, "lr": 0.0002999491698151174, "optimizer_lrs": [0.01999661132100783, 0.0002999491698151174], "gradient_norm": 0.4142177402973175, "seconds": 0.3344419410000228, "nonpadding_tokens_per_second": 254160.70647668623, "ranks": [{"compute_seconds": 0.3306792890000452, "nonpadding_tokens": 8600, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3312630500000182, "nonpadding_tokens": 13003, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3284858879999888, "nonpadding_tokens": 7176, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3315421590000369, "nonpadding_tokens": 7070, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3317209260000027, "nonpadding_tokens": 14043, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.32900458199998184, "nonpadding_tokens": 20375, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.33180253500000845, "nonpadding_tokens": 3315, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3320291190000262, "nonpadding_tokens": 11420, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035306500001297536, "prefetch": {"data_wait_seconds": 0.0013869159998449732, "consumed_batches": 145, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 695, "tokens": 63587267, "loss_tokens": 85985, "loss": 2.772784133860557, "lr": 0.00029994896868645583, "optimizer_lrs": [0.01999659791243039, 0.00029994896868645583], "gradient_norm": 0.37351998686790466, "seconds": 0.30805214700001216, "nonpadding_tokens_per_second": 279124.8197338375, "ranks": [{"compute_seconds": 0.30410971300000256, "nonpadding_tokens": 11266, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.30316705799998545, "nonpadding_tokens": 11611, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3037771339999722, "nonpadding_tokens": 9398, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.30112466899998935, "nonpadding_tokens": 11173, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3048182050000037, "nonpadding_tokens": 9660, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.30210723499999403, "nonpadding_tokens": 6928, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.29978683800004546, "nonpadding_tokens": 20441, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.30553785200004313, "nonpadding_tokens": 5508, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003409060000194586, "prefetch": {"data_wait_seconds": 0.0013956849998777443, "consumed_batches": 146, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 696, "tokens": 63640050, "loss_tokens": 52783, "loss": 2.812894007066669, "lr": 0.000299948845024354, "optimizer_lrs": [0.01999658966829027, 0.000299948845024354], "gradient_norm": 0.4564189612865448, "seconds": 0.33891422299996066, "nonpadding_tokens_per_second": 155741.47208335405, "ranks": [{"compute_seconds": 0.33445119800001066, "nonpadding_tokens": 10205, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.33565871100000777, "nonpadding_tokens": 4433, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3328285000000051, "nonpadding_tokens": 7020, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.33305736299996624, "nonpadding_tokens": 6625, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33139608099997986, "nonpadding_tokens": 12724, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3341766490000282, "nonpadding_tokens": 1537, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3353060779999737, "nonpadding_tokens": 8669, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.33637392200000704, "nonpadding_tokens": 1570, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042698099997551253, "prefetch": {"data_wait_seconds": 0.0014058969998842485, "consumed_batches": 147, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 697, "tokens": 63734391, "loss_tokens": 94341, "loss": 2.9531592838744554, "lr": 0.0002999486236258206, "optimizer_lrs": [0.01999657490838804, 0.0002999486236258206], "gradient_norm": 0.38282597064971924, "seconds": 0.32967881900003704, "nonpadding_tokens_per_second": 286160.3310948205, "ranks": [{"compute_seconds": 0.32501378000000614, "nonpadding_tokens": 14337, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3232535100000291, "nonpadding_tokens": 15836, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3237994380000373, "nonpadding_tokens": 16738, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.3264245929999561, "nonpadding_tokens": 14644, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32694232900001907, "nonpadding_tokens": 14568, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3252696350000406, "nonpadding_tokens": 4261, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3228750370000171, "nonpadding_tokens": 4059, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.32733080799999925, "nonpadding_tokens": 9898, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034377599996560093, "prefetch": {"data_wait_seconds": 0.0014158739998748615, "consumed_batches": 148, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 698, "tokens": 63799513, "loss_tokens": 65122, "loss": 3.738060390497835, "lr": 0.000299948470519302, "optimizer_lrs": [0.0199965647012868, 0.000299948470519302], "gradient_norm": 0.7411660552024841, "seconds": 0.261042046, "nonpadding_tokens_per_second": 249469.39007672347, "ranks": [{"compute_seconds": 0.25766174799997543, "nonpadding_tokens": 10595, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2584031910000135, "nonpadding_tokens": 10756, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2556229509999639, "nonpadding_tokens": 5386, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.258200214999988, "nonpadding_tokens": 11672, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25486433500003614, "nonpadding_tokens": 3112, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.25596395399998073, "nonpadding_tokens": 7096, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.25865927800003874, "nonpadding_tokens": 3676, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.25507134899999073, "nonpadding_tokens": 12829, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033710299999256677, "prefetch": {"data_wait_seconds": 0.0014240759998784824, "consumed_batches": 149, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 699, "tokens": 63854914, "loss_tokens": 55401, "loss": 2.9283132863125214, "lr": 0.0002999483400882611, "optimizer_lrs": [0.019996556005884077, 0.0002999483400882611], "gradient_norm": 0.7086631655693054, "seconds": 0.25800757700000077, "nonpadding_tokens_per_second": 214726.2520123579, "ranks": [{"compute_seconds": 0.25478676299997005, "nonpadding_tokens": 7617, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2547671899999955, "nonpadding_tokens": 3650, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2556134960000236, "nonpadding_tokens": 2958, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18356371456}, {"compute_seconds": 0.25221496900002194, "nonpadding_tokens": 7128, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2508267799999544, "nonpadding_tokens": 1525, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.25401766900000666, "nonpadding_tokens": 4282, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2504875810000158, "nonpadding_tokens": 15478, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2513029959999926, "nonpadding_tokens": 12763, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004477159999964897, "prefetch": {"data_wait_seconds": 0.0014349189998625889, "consumed_batches": 150, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 700, "tokens": 63933224, "loss_tokens": 78310, "loss": 3.0885263695568894, "lr": 0.0002999481554412025, "optimizer_lrs": [0.019996543696080168, 0.0002999481554412025], "gradient_norm": 0.48121729493141174, "seconds": 0.3526085760000228, "nonpadding_tokens_per_second": 222087.62160111198, "ranks": [{"compute_seconds": 0.3493882470000358, "nonpadding_tokens": 18880, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3503448490000096, "nonpadding_tokens": 4175, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3475690760000134, "nonpadding_tokens": 2176, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3497863080000343, "nonpadding_tokens": 24774, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34657278899999255, "nonpadding_tokens": 2919, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.34886526600001844, "nonpadding_tokens": 9814, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.34614505399997597, "nonpadding_tokens": 11646, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.34702409199996964, "nonpadding_tokens": 3926, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037331700002596335, "prefetch": {"data_wait_seconds": 0.0014448669998756714, "consumed_batches": 151, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 701, "tokens": 63991343, "loss_tokens": 58119, "loss": 3.116160377845455, "lr": 0.0002999480181895662, "optimizer_lrs": [0.01999653454597108, 0.0002999480181895662], "gradient_norm": 1.0513696670532227, "seconds": 0.28065133299998024, "nonpadding_tokens_per_second": 207086.13559303526, "ranks": [{"compute_seconds": 0.2779997940000385, "nonpadding_tokens": 4059, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.27793113699999594, "nonpadding_tokens": 3812, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2770656200000303, "nonpadding_tokens": 11378, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2774179140000115, "nonpadding_tokens": 4969, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2782455139999911, "nonpadding_tokens": 4618, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27674278500001037, "nonpadding_tokens": 4809, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2737824440000054, "nonpadding_tokens": 7409, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2743308979999597, "nonpadding_tokens": 17065, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003583899999739515, "prefetch": {"data_wait_seconds": 0.0014548689998861164, "consumed_batches": 152, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 702, "tokens": 64086943, "loss_tokens": 95600, "loss": 3.1260996338912133, "lr": 0.0002999477920296098, "optimizer_lrs": [0.019996519468640655, 0.0002999477920296098], "gradient_norm": 1.0310149192810059, "seconds": 0.3421318090000227, "nonpadding_tokens_per_second": 279424.4717537902, "ranks": [{"compute_seconds": 0.3389331849999735, "nonpadding_tokens": 25733, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.33878797500000246, "nonpadding_tokens": 18089, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3364427379999597, "nonpadding_tokens": 11772, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3384874480000235, "nonpadding_tokens": 11519, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33921218500000805, "nonpadding_tokens": 4622, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3379259409999804, "nonpadding_tokens": 5401, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3391729969999915, "nonpadding_tokens": 7226, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3397521169999891, "nonpadding_tokens": 11238, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037149000002045796, "prefetch": {"data_wait_seconds": 0.0014645879998766986, "consumed_batches": 153, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 703, "tokens": 64151111, "loss_tokens": 64168, "loss": 2.964557392313926, "lr": 0.00029994763995273236, "optimizer_lrs": [0.01999650933018216, 0.00029994763995273236], "gradient_norm": 0.7563593983650208, "seconds": 0.2789970970000013, "nonpadding_tokens_per_second": 229995.2246456518, "ranks": [{"compute_seconds": 0.24685355600001913, "nonpadding_tokens": 2877, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.27327443399997264, "nonpadding_tokens": 11813, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.273370731, "nonpadding_tokens": 12764, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27397424000002957, "nonpadding_tokens": 12416, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2723847400000068, "nonpadding_tokens": 5143, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27545715399998016, "nonpadding_tokens": 5347, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2766524509999613, "nonpadding_tokens": 4901, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2736321959999941, "nonpadding_tokens": 8907, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040901400001303045, "prefetch": {"data_wait_seconds": 0.0014737849998738284, "consumed_batches": 154, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 704, "tokens": 64219895, "loss_tokens": 68784, "loss": 3.140996634391719, "lr": 0.00029994747669046657, "optimizer_lrs": [0.019996498446031106, 0.00029994747669046657], "gradient_norm": 0.5586994290351868, "seconds": 0.30502469799995424, "nonpadding_tokens_per_second": 225503.05090380032, "ranks": [{"compute_seconds": 0.26060568500003, "nonpadding_tokens": 10532, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.26677749399999584, "nonpadding_tokens": 4764, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2677796600000306, "nonpadding_tokens": 2531, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27018379899999445, "nonpadding_tokens": 3610, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2662498799999753, "nonpadding_tokens": 19097, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.269562030999964, "nonpadding_tokens": 17528, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2702251539999452, "nonpadding_tokens": 6930, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.26725120399999014, "nonpadding_tokens": 3792, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.02520424599998705, "prefetch": {"data_wait_seconds": 0.0014839199998988306, "consumed_batches": 155, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 705, "tokens": 64282072, "loss_tokens": 62177, "loss": 3.379359530051305, "lr": 0.0002999473288915968, "optimizer_lrs": [0.019996488592773122, 0.0002999473288915968], "gradient_norm": 0.7102075815200806, "seconds": 0.2892778920000296, "nonpadding_tokens_per_second": 214938.65144728596, "ranks": [{"compute_seconds": 0.28700310300001775, "nonpadding_tokens": 7851, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2836439650000102, "nonpadding_tokens": 1529, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28605302400001165, "nonpadding_tokens": 12466, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.28678615100000115, "nonpadding_tokens": 2477, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2790747899999815, "nonpadding_tokens": 6016, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2823695589999602, "nonpadding_tokens": 10434, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2793035649999638, "nonpadding_tokens": 11522, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.27956836200002044, "nonpadding_tokens": 9882, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034820100000843013, "prefetch": {"data_wait_seconds": 0.0014927409998932717, "consumed_batches": 156, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 706, "tokens": 64348188, "loss_tokens": 66116, "loss": 3.5368256643626355, "lr": 0.0002999471715016979, "optimizer_lrs": [0.019996478100113194, 0.0002999471715016979], "gradient_norm": 0.9874654412269592, "seconds": 0.3199846680000178, "nonpadding_tokens_per_second": 206622.39979571872, "ranks": [{"compute_seconds": 0.3167355780000207, "nonpadding_tokens": 12696, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3170914229999653, "nonpadding_tokens": 4267, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3145349280000005, "nonpadding_tokens": 971, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3168252359999997, "nonpadding_tokens": 7948, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31312882699995725, "nonpadding_tokens": 15762, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.31604593899999145, "nonpadding_tokens": 12936, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31704477099998485, "nonpadding_tokens": 3771, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3175745249999977, "nonpadding_tokens": 7765, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004391639999994368, "prefetch": {"data_wait_seconds": 0.001501576999899612, "consumed_batches": 157, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 707, "tokens": 64415826, "loss_tokens": 67638, "loss": 3.7400130104379197, "lr": 0.0002999470102457353, "optimizer_lrs": [0.019996467349715687, 0.0002999470102457353], "gradient_norm": 0.5231658816337585, "seconds": 0.362676883000006, "nonpadding_tokens_per_second": 186496.58461964582, "ranks": [{"compute_seconds": 0.3593331739999712, "nonpadding_tokens": 16797, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3594990869999606, "nonpadding_tokens": 10096, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3565701279999871, "nonpadding_tokens": 3672, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.357502506000003, "nonpadding_tokens": 11084, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.360361768999951, "nonpadding_tokens": 6729, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3573345390000213, "nonpadding_tokens": 8405, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3556919300000345, "nonpadding_tokens": 4496, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.356464883000001, "nonpadding_tokens": 6359, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003463009999791211, "prefetch": {"data_wait_seconds": 0.0015096569998860332, "consumed_batches": 158, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 708, "tokens": 64472469, "loss_tokens": 56643, "loss": 3.2541046554737565, "lr": 0.0002999468750139582, "optimizer_lrs": [0.019996458334263882, 0.0002999468750139582], "gradient_norm": 1.339321255683899, "seconds": 0.24246202000000494, "nonpadding_tokens_per_second": 233615.97003934407, "ranks": [{"compute_seconds": 0.23837593500002185, "nonpadding_tokens": 10887, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.23965597500000513, "nonpadding_tokens": 15554, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23640894400000434, "nonpadding_tokens": 11061, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2393317629999956, "nonpadding_tokens": 2799, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23968691999999692, "nonpadding_tokens": 3426, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2384851870000375, "nonpadding_tokens": 3972, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.23957144800004926, "nonpadding_tokens": 4509, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2401813250000373, "nonpadding_tokens": 4435, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035476199997219737, "prefetch": {"data_wait_seconds": 0.0015190069999135858, "consumed_batches": 159, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 709, "tokens": 64530373, "loss_tokens": 57904, "loss": 3.25498454338215, "lr": 0.00029994673659351805, "optimizer_lrs": [0.019996449106234538, 0.00029994673659351805], "gradient_norm": 0.5788683891296387, "seconds": 0.2742232589999958, "nonpadding_tokens_per_second": 211156.41397873143, "ranks": [{"compute_seconds": 0.2718873590000044, "nonpadding_tokens": 6566, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.26822127500003035, "nonpadding_tokens": 7564, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26823764599998867, "nonpadding_tokens": 5424, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.268662618999997, "nonpadding_tokens": 5954, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2675542160000077, "nonpadding_tokens": 4232, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.26913315100000545, "nonpadding_tokens": 10369, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.26711752900001784, "nonpadding_tokens": 14394, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.26796439899999314, "nonpadding_tokens": 3401, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033583300000827876, "prefetch": {"data_wait_seconds": 0.0015273469999357303, "consumed_batches": 160, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 710, "tokens": 64589182, "loss_tokens": 58809, "loss": 3.3857325728204866, "lr": 0.0002999465958253561, "optimizer_lrs": [0.01999643972169041, 0.0002999465958253561], "gradient_norm": 0.6448019742965698, "seconds": 0.2562393320000069, "nonpadding_tokens_per_second": 229508.09128708785, "ranks": [{"compute_seconds": 0.25221731300001693, "nonpadding_tokens": 4140, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.24956960899999103, "nonpadding_tokens": 7181, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2524120940000216, "nonpadding_tokens": 6249, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.25087890899999366, "nonpadding_tokens": 5197, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24923059500002864, "nonpadding_tokens": 3183, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.251583815999993, "nonpadding_tokens": 8705, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2492784410000013, "nonpadding_tokens": 18263, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2538444030000164, "nonpadding_tokens": 5891, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040229300003602475, "prefetch": {"data_wait_seconds": 0.0015370169999897598, "consumed_batches": 161, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 711, "tokens": 64680739, "loss_tokens": 91557, "loss": 3.4064281676988104, "lr": 0.0002999463763002746, "optimizer_lrs": [0.019996425086684977, 0.0002999463763002746], "gradient_norm": 0.6002439856529236, "seconds": 0.31109115300000667, "nonpadding_tokens_per_second": 294309.23739576107, "ranks": [{"compute_seconds": 0.30803085700000565, "nonpadding_tokens": 21976, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3087906069999917, "nonpadding_tokens": 10662, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3060069620000263, "nonpadding_tokens": 11948, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3061961929999484, "nonpadding_tokens": 5415, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3051681110000004, "nonpadding_tokens": 4674, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.30760695200001464, "nonpadding_tokens": 5481, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30541564999998627, "nonpadding_tokens": 12549, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3072033290000036, "nonpadding_tokens": 18852, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034215900001299815, "prefetch": {"data_wait_seconds": 0.0015459030000357643, "consumed_batches": 162, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 712, "tokens": 64719896, "loss_tokens": 39157, "loss": 3.396846349311745, "lr": 0.00029994628227658507, "optimizer_lrs": [0.019996418818439006, 0.00029994628227658507], "gradient_norm": 0.59881591796875, "seconds": 0.2646270310000318, "nonpadding_tokens_per_second": 147970.52233108904, "ranks": [{"compute_seconds": 0.25919821899998396, "nonpadding_tokens": 4897, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.26145262300002514, "nonpadding_tokens": 1813, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2580964830000312, "nonpadding_tokens": 2836, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2617751989999988, "nonpadding_tokens": 13370, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26203004899997495, "nonpadding_tokens": 3964, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2603177070000129, "nonpadding_tokens": 5691, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2574419140000259, "nonpadding_tokens": 3593, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.26231010600002946, "nonpadding_tokens": 2993, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003437600000211205, "prefetch": {"data_wait_seconds": 0.0015552410000196915, "consumed_batches": 163, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 713, "tokens": 64795223, "loss_tokens": 75327, "loss": 3.140598656524221, "lr": 0.0002999461011700208, "optimizer_lrs": [0.019996406744668054, 0.0002999461011700208], "gradient_norm": 0.4492063820362091, "seconds": 0.24426495599999498, "nonpadding_tokens_per_second": 308382.3452759288, "ranks": [{"compute_seconds": 0.24161581200002047, "nonpadding_tokens": 9161, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.24177854099997376, "nonpadding_tokens": 6256, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23853581399998802, "nonpadding_tokens": 8258, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23912034299996776, "nonpadding_tokens": 4676, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2413040480000177, "nonpadding_tokens": 11742, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.24046131799997283, "nonpadding_tokens": 8306, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.24124135800002477, "nonpadding_tokens": 15103, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2419609689999902, "nonpadding_tokens": 11825, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033534400000689857, "prefetch": {"data_wait_seconds": 0.0015633239999743864, "consumed_batches": 164, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 714, "tokens": 64850772, "loss_tokens": 55549, "loss": 3.369877833084304, "lr": 0.000299945967419927, "optimizer_lrs": [0.019996397827995137, 0.000299945967419927], "gradient_norm": 0.5441182255744934, "seconds": 0.3214687720000029, "nonpadding_tokens_per_second": 172797.49959663113, "ranks": [{"compute_seconds": 0.3190378420000002, "nonpadding_tokens": 15999, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3160858189999658, "nonpadding_tokens": 2958, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31650595300004625, "nonpadding_tokens": 12289, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3191304110000033, "nonpadding_tokens": 6442, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3157973450000213, "nonpadding_tokens": 3393, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3182628899999713, "nonpadding_tokens": 1555, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3155514979999907, "nonpadding_tokens": 7063, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3162911389999863, "nonpadding_tokens": 5850, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039185200000702025, "prefetch": {"data_wait_seconds": 0.0015722739999688429, "consumed_batches": 165, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 715, "tokens": 64900373, "loss_tokens": 49601, "loss": 3.0187493069696174, "lr": 0.00029994584785129723, "optimizer_lrs": [0.01999638985675315, 0.00029994584785129723], "gradient_norm": 0.6938471794128418, "seconds": 0.28007847000003494, "nonpadding_tokens_per_second": 177096.79719399285, "ranks": [{"compute_seconds": 0.2762599139999793, "nonpadding_tokens": 12041, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2772303899999997, "nonpadding_tokens": 6096, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2744853160000389, "nonpadding_tokens": 3611, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2749184360000072, "nonpadding_tokens": 10914, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27765361399997346, "nonpadding_tokens": 7958, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27614831600004663, "nonpadding_tokens": 4500, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.27291156600000477, "nonpadding_tokens": 1780, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2743278400000122, "nonpadding_tokens": 2701, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003955180000048131, "prefetch": {"data_wait_seconds": 0.0015822069999700261, "consumed_batches": 166, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 716, "tokens": 64965102, "loss_tokens": 64729, "loss": 3.1706296636747053, "lr": 0.00029994569161625605, "optimizer_lrs": [0.019996379441083737, 0.00029994569161625605], "gradient_norm": 0.5179520845413208, "seconds": 0.2567002080000407, "nonpadding_tokens_per_second": 252157.9569580627, "ranks": [{"compute_seconds": 0.22262191800001574, "nonpadding_tokens": 2723, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.21984250799999927, "nonpadding_tokens": 5403, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22044952999999623, "nonpadding_tokens": 4452, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.22300213400001212, "nonpadding_tokens": 7267, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21966438399999788, "nonpadding_tokens": 18233, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2222083749999797, "nonpadding_tokens": 6404, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.21952769199998556, "nonpadding_tokens": 13814, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2242739819999997, "nonpadding_tokens": 6433, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.029122633999975278, "prefetch": {"data_wait_seconds": 0.001590306999972313, "consumed_batches": 167, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 717, "tokens": 65005840, "loss_tokens": 40738, "loss": 3.290462835681673, "lr": 0.00029994559317242435, "optimizer_lrs": [0.019996372878161625, 0.00029994559317242435], "gradient_norm": 0.5845751166343689, "seconds": 0.24132135299998936, "nonpadding_tokens_per_second": 168812.2476256869, "ranks": [{"compute_seconds": 0.23894545799998923, "nonpadding_tokens": 4184, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2352773219999449, "nonpadding_tokens": 6169, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23599527199996828, "nonpadding_tokens": 5054, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23636630699996886, "nonpadding_tokens": 3046, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2350064899999893, "nonpadding_tokens": 1554, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.23746431099999654, "nonpadding_tokens": 8764, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2342002909999792, "nonpadding_tokens": 1968, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.23550230199998623, "nonpadding_tokens": 9999, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003726439999809372, "prefetch": {"data_wait_seconds": 0.0016004289999642651, "consumed_batches": 168, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 718, "tokens": 65080281, "loss_tokens": 74441, "loss": 3.174629278556172, "lr": 0.0002999454130546906, "optimizer_lrs": [0.019996360870312705, 0.0002999454130546906], "gradient_norm": 0.4449594020843506, "seconds": 0.241119733000005, "nonpadding_tokens_per_second": 308730.4347670228, "ranks": [{"compute_seconds": 0.2381401239999832, "nonpadding_tokens": 7275, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.23788737199998877, "nonpadding_tokens": 7521, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2346517090000475, "nonpadding_tokens": 5539, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2377242950000209, "nonpadding_tokens": 2658, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23819647400000576, "nonpadding_tokens": 16208, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.23526953000003914, "nonpadding_tokens": 10413, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.233809448000045, "nonpadding_tokens": 9672, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.23882788000003075, "nonpadding_tokens": 15155, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034114499999304826, "prefetch": {"data_wait_seconds": 0.0016095649999243733, "consumed_batches": 169, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 719, "tokens": 65147204, "loss_tokens": 66923, "loss": 3.141089619786919, "lr": 0.00029994525087350977, "optimizer_lrs": [0.019996350058233987, 0.00029994525087350977], "gradient_norm": 0.7903457283973694, "seconds": 0.2526626739999642, "nonpadding_tokens_per_second": 264870.94013740023, "ranks": [{"compute_seconds": 0.25027010400003746, "nonpadding_tokens": 3402, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2500357349999831, "nonpadding_tokens": 10399, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24738513599999123, "nonpadding_tokens": 12091, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.24955976100000044, "nonpadding_tokens": 7248, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24610734100002674, "nonpadding_tokens": 3750, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2472155660000226, "nonpadding_tokens": 9200, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.24612849199996845, "nonpadding_tokens": 11899, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.24686648300001934, "nonpadding_tokens": 8934, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003924279999978353, "prefetch": {"data_wait_seconds": 0.001617662999947811, "consumed_batches": 170, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 720, "tokens": 65205858, "loss_tokens": 58654, "loss": 3.3058729477103013, "lr": 0.0002999451085336442, "optimizer_lrs": [0.01999634056890961, 0.0002999451085336442], "gradient_norm": 0.6143413186073303, "seconds": 0.3204695210000068, "nonpadding_tokens_per_second": 183025.20569498636, "ranks": [{"compute_seconds": 0.31759913299998743, "nonpadding_tokens": 6609, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.31797108799997886, "nonpadding_tokens": 4926, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31715614799998093, "nonpadding_tokens": 3018, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.31551708099999587, "nonpadding_tokens": 10427, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3170115080000073, "nonpadding_tokens": 9376, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3152945290000275, "nonpadding_tokens": 14416, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31801602000001594, "nonpadding_tokens": 5061, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.31455819200004953, "nonpadding_tokens": 4821, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003924279999978353, "prefetch": {"data_wait_seconds": 0.0016277969999833886, "consumed_batches": 171, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 721, "tokens": 65256178, "loss_tokens": 50320, "loss": 3.134912497515898, "lr": 0.00029994498627125164, "optimizer_lrs": [0.019996332418083445, 0.00029994498627125164], "gradient_norm": 0.8076105713844299, "seconds": 0.27107182899999316, "nonpadding_tokens_per_second": 185633.4543712444, "ranks": [{"compute_seconds": 0.2680697000000123, "nonpadding_tokens": 2644, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2641019829999891, "nonpadding_tokens": 2719, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26698395900001515, "nonpadding_tokens": 5648, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2672657699999945, "nonpadding_tokens": 5585, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26367884499995853, "nonpadding_tokens": 3029, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.26676879900003314, "nonpadding_tokens": 15969, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.26408710499998733, "nonpadding_tokens": 3880, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.26868501299998115, "nonpadding_tokens": 10846, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039213499997003964, "prefetch": {"data_wait_seconds": 0.001637071999951445, "consumed_batches": 172, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 722, "tokens": 65300341, "loss_tokens": 44163, "loss": 2.9777521624436747, "lr": 0.00029994487885646657, "optimizer_lrs": [0.019996325257097772, 0.00029994487885646657], "gradient_norm": 0.9203444719314575, "seconds": 0.3097996380000154, "nonpadding_tokens_per_second": 142553.4267409241, "ranks": [{"compute_seconds": 0.30710068300004423, "nonpadding_tokens": 12346, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3072421850000069, "nonpadding_tokens": 6364, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30388346000000865, "nonpadding_tokens": 3030, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3065954030000171, "nonpadding_tokens": 6091, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3030035870000347, "nonpadding_tokens": 1824, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3045978320000131, "nonpadding_tokens": 9472, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3030011299999842, "nonpadding_tokens": 2984, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.30738134500001024, "nonpadding_tokens": 2052, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004026590000307806, "prefetch": {"data_wait_seconds": 0.001655739999947059, "consumed_batches": 173, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 723, "tokens": 65345818, "loss_tokens": 45477, "loss": 3.3358871654902478, "lr": 0.0002999447681362636, "optimizer_lrs": [0.01999631787575091, 0.0002999447681362636], "gradient_norm": 0.5514345169067383, "seconds": 0.28739625299999716, "nonpadding_tokens_per_second": 158237.97118190143, "ranks": [{"compute_seconds": 0.2827193809999926, "nonpadding_tokens": 12939, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.28067622800000436, "nonpadding_tokens": 5781, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28335546800002476, "nonpadding_tokens": 4359, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.28358020299998543, "nonpadding_tokens": 3875, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2800239489999967, "nonpadding_tokens": 1559, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2831447549999666, "nonpadding_tokens": 3713, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2799773659999687, "nonpadding_tokens": 7644, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2848845519999941, "nonpadding_tokens": 5607, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003584899999964364, "prefetch": {"data_wait_seconds": 0.0016646389999124267, "consumed_batches": 174, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 724, "tokens": 65415992, "loss_tokens": 70174, "loss": 3.5735168117821416, "lr": 0.00029994459706979537, "optimizer_lrs": [0.019996306471319693, 0.00029994459706979537], "gradient_norm": 0.472036212682724, "seconds": 0.2986649929999885, "nonpadding_tokens_per_second": 234958.90594718177, "ranks": [{"compute_seconds": 0.29456017199998996, "nonpadding_tokens": 11473, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.29198911600002475, "nonpadding_tokens": 14278, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29494882500000585, "nonpadding_tokens": 10235, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2926892300000077, "nonpadding_tokens": 6371, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2913925760000211, "nonpadding_tokens": 8078, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.29311219600003824, "nonpadding_tokens": 2535, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2955779389999975, "nonpadding_tokens": 5599, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2962742959999787, "nonpadding_tokens": 11605, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043127099996809193, "prefetch": {"data_wait_seconds": 0.001674096999877861, "consumed_batches": 175, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 725, "tokens": 65479026, "loss_tokens": 63034, "loss": 3.3471888683091664, "lr": 0.00029994444318338063, "optimizer_lrs": [0.019996296212225376, 0.00029994444318338063], "gradient_norm": 0.5209078192710876, "seconds": 0.33901047299997344, "nonpadding_tokens_per_second": 185935.2587021845, "ranks": [{"compute_seconds": 0.3337346160000152, "nonpadding_tokens": 11614, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3325803980000046, "nonpadding_tokens": 2002, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33464197800003603, "nonpadding_tokens": 6384, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.32979540899998483, "nonpadding_tokens": 12660, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33623055600003227, "nonpadding_tokens": 4692, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.331118435999997, "nonpadding_tokens": 11442, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3281764550000048, "nonpadding_tokens": 3975, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.336531358000002, "nonpadding_tokens": 10265, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004053079999835063, "prefetch": {"data_wait_seconds": 0.0016829269998908103, "consumed_batches": 176, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 726, "tokens": 65574717, "loss_tokens": 95691, "loss": 3.492519868639684, "lr": 0.0002999442091628162, "optimizer_lrs": [0.019996280610854414, 0.0002999442091628162], "gradient_norm": 0.893934965133667, "seconds": 0.3660316529999932, "nonpadding_tokens_per_second": 261428.21041764325, "ranks": [{"compute_seconds": 0.36263620000005403, "nonpadding_tokens": 25585, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3629654300000311, "nonpadding_tokens": 9162, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3596779260000176, "nonpadding_tokens": 9662, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3605582939999863, "nonpadding_tokens": 17631, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3590201659999934, "nonpadding_tokens": 6899, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3598552160000281, "nonpadding_tokens": 11349, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.36256315399998584, "nonpadding_tokens": 9815, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3632692190000171, "nonpadding_tokens": 5588, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005261099999529506, "prefetch": {"data_wait_seconds": 0.0016916879998944978, "consumed_batches": 177, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 727, "tokens": 65629987, "loss_tokens": 55270, "loss": 3.162846028586937, "lr": 0.00029994407377124417, "optimizer_lrs": [0.019996271584749613, 0.00029994407377124417], "gradient_norm": 0.6664204597473145, "seconds": 0.2944643170000063, "nonpadding_tokens_per_second": 187696.7659887932, "ranks": [{"compute_seconds": 0.29190523700003723, "nonpadding_tokens": 11758, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2879253810000364, "nonpadding_tokens": 10552, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.288212370999986, "nonpadding_tokens": 8642, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.28861491700001807, "nonpadding_tokens": 7279, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2919003860000089, "nonpadding_tokens": 3609, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2900042499999813, "nonpadding_tokens": 2826, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.28731373099998336, "nonpadding_tokens": 8941, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2919268969999962, "nonpadding_tokens": 1663, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004091440000024704, "prefetch": {"data_wait_seconds": 0.0017010799998757875, "consumed_batches": 178, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 728, "tokens": 65681723, "loss_tokens": 51736, "loss": 3.2095122955968765, "lr": 0.0002999439468880476, "optimizer_lrs": [0.019996263125869843, 0.0002999439468880476], "gradient_norm": 0.6620056629180908, "seconds": 0.3128055860000245, "nonpadding_tokens_per_second": 165393.46583150837, "ranks": [{"compute_seconds": 0.31037689299995463, "nonpadding_tokens": 11570, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3061743149999643, "nonpadding_tokens": 9408, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30666120900002625, "nonpadding_tokens": 4204, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.30910835800000314, "nonpadding_tokens": 5460, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30593737499998497, "nonpadding_tokens": 10397, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3086546430000112, "nonpadding_tokens": 1703, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30988429400002815, "nonpadding_tokens": 2017, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.30671159700000317, "nonpadding_tokens": 6977, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041499299999259165, "prefetch": {"data_wait_seconds": 0.0017114229998611563, "consumed_batches": 179, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 729, "tokens": 65752511, "loss_tokens": 70788, "loss": 3.511619200994519, "lr": 0.0002999437730466923, "optimizer_lrs": [0.019996251536446154, 0.0002999437730466923], "gradient_norm": 0.699378252029419, "seconds": 0.2982538059999911, "nonpadding_tokens_per_second": 237341.48089966745, "ranks": [{"compute_seconds": 0.29592825499997844, "nonpadding_tokens": 10546, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2916250079999827, "nonpadding_tokens": 6884, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29263814700004787, "nonpadding_tokens": 2197, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.29434861099997534, "nonpadding_tokens": 8654, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.295647506000023, "nonpadding_tokens": 3253, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.29369138399999883, "nonpadding_tokens": 17185, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2948250119999898, "nonpadding_tokens": 18273, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.29536708200004114, "nonpadding_tokens": 3796, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039245200002824276, "prefetch": {"data_wait_seconds": 0.0017202129998423743, "consumed_batches": 180, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 730, "tokens": 65822643, "loss_tokens": 70132, "loss": 3.244946582872298, "lr": 0.00029994360055097294, "optimizer_lrs": [0.01999624003673153, 0.00029994360055097294], "gradient_norm": 0.560990035533905, "seconds": 0.27350637899996855, "nonpadding_tokens_per_second": 256418.15103701132, "ranks": [{"compute_seconds": 0.26954388599995127, "nonpadding_tokens": 4665, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2712147619999996, "nonpadding_tokens": 5815, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2680944210000007, "nonpadding_tokens": 9846, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27047858899999255, "nonpadding_tokens": 10883, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2671872619999931, "nonpadding_tokens": 4905, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2681990199999973, "nonpadding_tokens": 4195, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.26715058700000327, "nonpadding_tokens": 18399, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.26769606599998497, "nonpadding_tokens": 11424, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034525199998824974, "prefetch": {"data_wait_seconds": 0.0017296599998672946, "consumed_batches": 181, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 731, "tokens": 65879192, "loss_tokens": 56549, "loss": 3.2762128861695166, "lr": 0.0002999434612714634, "optimizer_lrs": [0.019996230751430893, 0.0002999434612714634], "gradient_norm": 0.5077223181724548, "seconds": 0.28555247500003134, "nonpadding_tokens_per_second": 198033.65388443504, "ranks": [{"compute_seconds": 0.2816170880000186, "nonpadding_tokens": 3938, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2801623440000185, "nonpadding_tokens": 7129, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28057974500001137, "nonpadding_tokens": 6377, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2827262089999749, "nonpadding_tokens": 5016, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27942083799996453, "nonpadding_tokens": 3732, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.28237829799996916, "nonpadding_tokens": 17993, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.283232508000026, "nonpadding_tokens": 7367, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.28051369299998896, "nonpadding_tokens": 4997, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035801600000695544, "prefetch": {"data_wait_seconds": 0.0017386989998868785, "consumed_batches": 182, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 732, "tokens": 65969963, "loss_tokens": 90771, "loss": 3.0011440190148835, "lr": 0.000299943237344514, "optimizer_lrs": [0.0199962158229676, 0.000299943237344514], "gradient_norm": 0.7461553812026978, "seconds": 0.3128306950000024, "nonpadding_tokens_per_second": 290160.145570112, "ranks": [{"compute_seconds": 0.3073301370000081, "nonpadding_tokens": 3191, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.306478310999978, "nonpadding_tokens": 10413, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3092635829999608, "nonpadding_tokens": 11879, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.30938163799999074, "nonpadding_tokens": 18594, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3059720789999574, "nonpadding_tokens": 18066, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3074233980000258, "nonpadding_tokens": 20644, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30978801199995587, "nonpadding_tokens": 4258, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.310532999999964, "nonpadding_tokens": 3726, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003615229999809344, "prefetch": {"data_wait_seconds": 0.001747122999930184, "consumed_batches": 183, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 733, "tokens": 66030060, "loss_tokens": 60097, "loss": 3.199422392132719, "lr": 0.0002999430888451453, "optimizer_lrs": [0.01999620592300969, 0.0002999430888451453], "gradient_norm": 0.6368494629859924, "seconds": 0.270135941000035, "nonpadding_tokens_per_second": 222469.47139844752, "ranks": [{"compute_seconds": 0.2666394530000389, "nonpadding_tokens": 12954, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.26430663700000423, "nonpadding_tokens": 6326, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26712152999999716, "nonpadding_tokens": 3212, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2678523469999732, "nonpadding_tokens": 10960, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2637691909999944, "nonpadding_tokens": 8929, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2671246760000372, "nonpadding_tokens": 6221, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.263887053000019, "nonpadding_tokens": 6391, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.26453117799997017, "nonpadding_tokens": 5104, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003352110000491848, "prefetch": {"data_wait_seconds": 0.0017546109999244663, "consumed_batches": 184, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 734, "tokens": 66084520, "loss_tokens": 54460, "loss": 3.181332055637165, "lr": 0.0002999429541072559, "optimizer_lrs": [0.01999619694048373, 0.0002999429541072559], "gradient_norm": 0.5562136769294739, "seconds": 0.30603984300000775, "nonpadding_tokens_per_second": 177950.68598306208, "ranks": [{"compute_seconds": 0.3026360830000385, "nonpadding_tokens": 9919, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3037422649999826, "nonpadding_tokens": 7250, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3007535260000509, "nonpadding_tokens": 10618, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3030516580000153, "nonpadding_tokens": 6614, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2991866509999568, "nonpadding_tokens": 3657, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.30185868800003846, "nonpadding_tokens": 9081, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30284433399998534, "nonpadding_tokens": 3660, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2998416110000335, "nonpadding_tokens": 3661, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035538700001325196, "prefetch": {"data_wait_seconds": 0.001763866999922925, "consumed_batches": 185, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 735, "tokens": 66137155, "loss_tokens": 52635, "loss": 3.133968604540705, "lr": 0.0002999428237331876, "optimizer_lrs": [0.019996188248879176, 0.0002999428237331876], "gradient_norm": 0.6568617224693298, "seconds": 0.2432963109999946, "nonpadding_tokens_per_second": 216341.13474084352, "ranks": [{"compute_seconds": 0.24018697100001418, "nonpadding_tokens": 4142, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.24066679499998145, "nonpadding_tokens": 1326, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2375286390000042, "nonpadding_tokens": 12195, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23797852800004193, "nonpadding_tokens": 6405, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23617232199995897, "nonpadding_tokens": 13932, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2380145689999722, "nonpadding_tokens": 2580, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2369308300000057, "nonpadding_tokens": 6292, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.24098567299995466, "nonpadding_tokens": 5763, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004170839999915188, "prefetch": {"data_wait_seconds": 0.001773570999901608, "consumed_batches": 186, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 736, "tokens": 66211872, "loss_tokens": 74717, "loss": 3.450901610409947, "lr": 0.0002999426384076941, "optimizer_lrs": [0.019996175893846273, 0.0002999426384076941], "gradient_norm": 0.9734340310096741, "seconds": 0.2729928729999642, "nonpadding_tokens_per_second": 273695.7898531285, "ranks": [{"compute_seconds": 0.2705738889999907, "nonpadding_tokens": 4510, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2705684399999768, "nonpadding_tokens": 9886, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26974181799999997, "nonpadding_tokens": 14056, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2680927569999767, "nonpadding_tokens": 9166, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27054134899998417, "nonpadding_tokens": 17832, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2692091209999603, "nonpadding_tokens": 6202, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.26651251000004095, "nonpadding_tokens": 9119, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2707671370000071, "nonpadding_tokens": 3946, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034543000003850466, "prefetch": {"data_wait_seconds": 0.0017821709999026325, "consumed_batches": 187, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 737, "tokens": 66262358, "loss_tokens": 50486, "loss": 3.548091302539318, "lr": 0.00029994251301421985, "optimizer_lrs": [0.019996167534281323, 0.00029994251301421985], "gradient_norm": 0.5414968132972717, "seconds": 0.24757715999999164, "nonpadding_tokens_per_second": 203920.26469647567, "ranks": [{"compute_seconds": 0.24465876200002867, "nonpadding_tokens": 3243, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2445459539999888, "nonpadding_tokens": 4501, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24357889799995291, "nonpadding_tokens": 15326, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.24203136500000255, "nonpadding_tokens": 4334, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24453282799998988, "nonpadding_tokens": 3635, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2429452360000255, "nonpadding_tokens": 3520, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.24038989299998548, "nonpadding_tokens": 9919, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.24532218899997815, "nonpadding_tokens": 6008, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034648700000161625, "prefetch": {"data_wait_seconds": 0.0017910809998653576, "consumed_batches": 188, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 738, "tokens": 66341949, "loss_tokens": 79591, "loss": 3.2917664607179202, "lr": 0.0002999423150538654, "optimizer_lrs": [0.019996154336924362, 0.0002999423150538654], "gradient_norm": 1.3418958187103271, "seconds": 0.2695219990000055, "nonpadding_tokens_per_second": 295304.28052367765, "ranks": [{"compute_seconds": 0.2659107950000248, "nonpadding_tokens": 8803, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.26633073299996113, "nonpadding_tokens": 10751, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26645017000004145, "nonpadding_tokens": 3648, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.26420876899999257, "nonpadding_tokens": 19379, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26728802200000246, "nonpadding_tokens": 9814, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2655043990000081, "nonpadding_tokens": 21159, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2623382009999773, "nonpadding_tokens": 3390, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2632689360000313, "nonpadding_tokens": 2647, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003553939999960676, "prefetch": {"data_wait_seconds": 0.001800076999828093, "consumed_batches": 189, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 739, "tokens": 66406757, "loss_tokens": 64808, "loss": 3.309518115047525, "lr": 0.0002999421536108128, "optimizer_lrs": [0.019996143574054187, 0.0002999421536108128], "gradient_norm": 0.6088560223579407, "seconds": 0.3608230759999742, "nonpadding_tokens_per_second": 179611.57229313304, "ranks": [{"compute_seconds": 0.3579448690000504, "nonpadding_tokens": 9176, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3548947770000268, "nonpadding_tokens": 3545, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3578104279999934, "nonpadding_tokens": 2584, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3555509410000468, "nonpadding_tokens": 9804, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35852217600000813, "nonpadding_tokens": 18034, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3571429460000104, "nonpadding_tokens": 4769, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.35407005800004754, "nonpadding_tokens": 4345, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3552024269999947, "nonpadding_tokens": 12551, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003742859999533721, "prefetch": {"data_wait_seconds": 0.0018086889998158995, "consumed_batches": 190, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 740, "tokens": 66470265, "loss_tokens": 63508, "loss": 3.270674560685268, "lr": 0.0002999419951873766, "optimizer_lrs": [0.019996133012491774, 0.0002999419951873766], "gradient_norm": 0.5366506576538086, "seconds": 0.26953831900004843, "nonpadding_tokens_per_second": 235617.70450897777, "ranks": [{"compute_seconds": 0.25011367700000164, "nonpadding_tokens": 7656, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2661725419999925, "nonpadding_tokens": 10181, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26465825899998663, "nonpadding_tokens": 6724, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2626555659999781, "nonpadding_tokens": 9213, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2615501540000196, "nonpadding_tokens": 5705, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.26441906100001233, "nonpadding_tokens": 8514, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2615469500000245, "nonpadding_tokens": 10255, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2624325660000295, "nonpadding_tokens": 5260, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003625889999625542, "prefetch": {"data_wait_seconds": 0.0018174719998143019, "consumed_batches": 191, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 741, "tokens": 66533902, "loss_tokens": 63637, "loss": 3.460306356757861, "lr": 0.0002999418362248906, "optimizer_lrs": [0.01999612241499271, 0.0002999418362248906], "gradient_norm": 0.9245830774307251, "seconds": 0.3035049430000072, "nonpadding_tokens_per_second": 209673.6856111055, "ranks": [{"compute_seconds": 0.2734032860000184, "nonpadding_tokens": 11762, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2893191040000147, "nonpadding_tokens": 8215, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28994307099998196, "nonpadding_tokens": 1356, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2901331429999914, "nonpadding_tokens": 13003, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2930214550000301, "nonpadding_tokens": 4754, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.29035975399995095, "nonpadding_tokens": 4480, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2886069499999735, "nonpadding_tokens": 10250, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.29354078199997957, "nonpadding_tokens": 9817, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004100159999893549, "prefetch": {"data_wait_seconds": 0.0018267819998101231, "consumed_batches": 192, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 742, "tokens": 66577937, "loss_tokens": 44035, "loss": 3.2359153230384923, "lr": 0.00029994172610004784, "optimizer_lrs": [0.019996115073336523, 0.00029994172610004784], "gradient_norm": 0.5501879453659058, "seconds": 0.2400967169999717, "nonpadding_tokens_per_second": 183405.25664082775, "ranks": [{"compute_seconds": 0.23710930700002564, "nonpadding_tokens": 3506, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.23766140600002927, "nonpadding_tokens": 5214, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2367869899999846, "nonpadding_tokens": 11060, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23748075299999982, "nonpadding_tokens": 5489, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2333810370000151, "nonpadding_tokens": 7628, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2363846260000173, "nonpadding_tokens": 3592, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.23370676599995477, "nonpadding_tokens": 4193, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.23780856100000847, "nonpadding_tokens": 3353, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003602159999900323, "prefetch": {"data_wait_seconds": 0.0018358619997940195, "consumed_batches": 193, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 743, "tokens": 66662107, "loss_tokens": 84170, "loss": 3.0507718753712725, "lr": 0.0002999415153139222, "optimizer_lrs": [0.019996101020928147, 0.0002999415153139222], "gradient_norm": 0.7265169024467468, "seconds": 0.2888340320000111, "nonpadding_tokens_per_second": 291413.02850349975, "ranks": [{"compute_seconds": 0.28495889900000293, "nonpadding_tokens": 13273, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.28554193399997985, "nonpadding_tokens": 14099, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2830406549999793, "nonpadding_tokens": 14649, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.28524579900005165, "nonpadding_tokens": 11537, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28593100399996274, "nonpadding_tokens": 16236, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.28439877499999966, "nonpadding_tokens": 3990, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2857556839999802, "nonpadding_tokens": 3549, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2865253989999701, "nonpadding_tokens": 6837, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037634799997476875, "prefetch": {"data_wait_seconds": 0.0018441689998240918, "consumed_batches": 194, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 744, "tokens": 66725211, "loss_tokens": 63104, "loss": 3.196768677500634, "lr": 0.00029994135703363906, "optimizer_lrs": [0.01999609046890927, 0.00029994135703363906], "gradient_norm": 0.6142001152038574, "seconds": 0.31084497400001965, "nonpadding_tokens_per_second": 203007.94697744097, "ranks": [{"compute_seconds": 0.3084076460000347, "nonpadding_tokens": 8627, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3086048790000291, "nonpadding_tokens": 10828, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30603174300000546, "nonpadding_tokens": 2955, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.30813691599996673, "nonpadding_tokens": 8969, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3047344080000016, "nonpadding_tokens": 15513, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3060927779999929, "nonpadding_tokens": 6354, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3044739729999719, "nonpadding_tokens": 3708, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3053057149999745, "nonpadding_tokens": 6150, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003676729999710915, "prefetch": {"data_wait_seconds": 0.0018536969997882125, "consumed_batches": 195, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 745, "tokens": 66823420, "loss_tokens": 98209, "loss": 3.1779098020548013, "lr": 0.00029994111027604076, "optimizer_lrs": [0.01999607401840272, 0.00029994111027604076], "gradient_norm": 0.5348736047744751, "seconds": 0.3174657889999821, "nonpadding_tokens_per_second": 309353.0181924753, "ranks": [{"compute_seconds": 0.31383875000000216, "nonpadding_tokens": 21623, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3140117830000122, "nonpadding_tokens": 14014, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3143081219999999, "nonpadding_tokens": 24625, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3130558189999988, "nonpadding_tokens": 5596, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31521974100002126, "nonpadding_tokens": 3855, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3126823540000032, "nonpadding_tokens": 2691, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.31138843399997995, "nonpadding_tokens": 21941, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3117733270000258, "nonpadding_tokens": 3864, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035373599996546545, "prefetch": {"data_wait_seconds": 0.001863098999777435, "consumed_batches": 196, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 746, "tokens": 66876812, "loss_tokens": 53392, "loss": 3.144796388035661, "lr": 0.0002999409759072341, "optimizer_lrs": [0.019996065060482276, 0.0002999409759072341], "gradient_norm": 0.8952217698097229, "seconds": 0.31907939800004215, "nonpadding_tokens_per_second": 167331.39254572915, "ranks": [{"compute_seconds": 0.3168870689999608, "nonpadding_tokens": 2356, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3130550330000119, "nonpadding_tokens": 3685, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3135698780000098, "nonpadding_tokens": 5634, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.31645485300003884, "nonpadding_tokens": 4337, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3166194899999937, "nonpadding_tokens": 10683, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3154775090000044, "nonpadding_tokens": 6481, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3125422649999905, "nonpadding_tokens": 7217, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3131093880000435, "nonpadding_tokens": 12999, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003493599999728758, "prefetch": {"data_wait_seconds": 0.0018716259997546558, "consumed_batches": 197, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 747, "tokens": 66925343, "loss_tokens": 48531, "loss": 3.37869931074983, "lr": 0.00029994085363903273, "optimizer_lrs": [0.01999605690926885, 0.00029994085363903273], "gradient_norm": 0.5796935558319092, "seconds": 0.27865207199999986, "nonpadding_tokens_per_second": 174163.42771712827, "ranks": [{"compute_seconds": 0.2754627730000152, "nonpadding_tokens": 6667, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.27450712000000976, "nonpadding_tokens": 9077, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27508506400005217, "nonpadding_tokens": 2401, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27626954200002274, "nonpadding_tokens": 2303, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27089951599998585, "nonpadding_tokens": 2625, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2735269699999776, "nonpadding_tokens": 2412, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2755066199999874, "nonpadding_tokens": 12051, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2719734849999895, "nonpadding_tokens": 10995, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003368089999753465, "prefetch": {"data_wait_seconds": 0.0018796339997635414, "consumed_batches": 198, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 748, "tokens": 66987714, "loss_tokens": 62371, "loss": 3.320359471950105, "lr": 0.00029994069631683596, "optimizer_lrs": [0.019996046421122397, 0.00029994069631683596], "gradient_norm": 0.4808003902435303, "seconds": 0.29819917999998324, "nonpadding_tokens_per_second": 209158.858183324, "ranks": [{"compute_seconds": 0.29579573300003403, "nonpadding_tokens": 4851, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.29217170800001213, "nonpadding_tokens": 7390, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29514802100004545, "nonpadding_tokens": 2525, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.29353104999995594, "nonpadding_tokens": 20048, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2916497800000002, "nonpadding_tokens": 5558, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.29455486299997347, "nonpadding_tokens": 2210, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2955150220000178, "nonpadding_tokens": 7468, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2922562499999799, "nonpadding_tokens": 12321, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040817299998252565, "prefetch": {"data_wait_seconds": 0.0018905289997661612, "consumed_batches": 199, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 749, "tokens": 67059077, "loss_tokens": 71363, "loss": 3.3664854336280707, "lr": 0.0002999405160573133, "optimizer_lrs": [0.01999603440382089, 0.0002999405160573133], "gradient_norm": 0.48636701703071594, "seconds": 0.29900744100001475, "nonpadding_tokens_per_second": 238666.30128444356, "ranks": [{"compute_seconds": 0.27245688099998233, "nonpadding_tokens": 3400, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2761164379999741, "nonpadding_tokens": 18194, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2786020409999992, "nonpadding_tokens": 11783, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2792579339999861, "nonpadding_tokens": 12928, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2756264380000175, "nonpadding_tokens": 5181, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27854680199999393, "nonpadding_tokens": 14177, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.27548572699998886, "nonpadding_tokens": 1893, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2760124010000027, "nonpadding_tokens": 3807, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.004881883999985348, "prefetch": {"data_wait_seconds": 0.001899068999776432, "consumed_batches": 200, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 750, "tokens": 67156395, "loss_tokens": 97318, "loss": 3.2435848198688833, "lr": 0.0002999402697959322, "optimizer_lrs": [0.01999601798639548, 0.0002999402697959322], "gradient_norm": 0.4903172552585602, "seconds": 0.2818143389999932, "nonpadding_tokens_per_second": 345326.6442911634, "ranks": [{"compute_seconds": 0.2765309650000063, "nonpadding_tokens": 2426, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2783425829999828, "nonpadding_tokens": 8871, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27550218799996173, "nonpadding_tokens": 13245, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2759915140000544, "nonpadding_tokens": 21070, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2785419389999788, "nonpadding_tokens": 13657, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.275527784000019, "nonpadding_tokens": 12344, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.27945355700001073, "nonpadding_tokens": 15053, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2791306959999815, "nonpadding_tokens": 10652, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003913429999897744, "prefetch": {"data_wait_seconds": 0.0019093619997789801, "consumed_batches": 201, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 751, "tokens": 67223305, "loss_tokens": 66910, "loss": 3.4924235913914212, "lr": 0.00029994010018638025, "optimizer_lrs": [0.01999600667909202, 0.00029994010018638025], "gradient_norm": 0.46846720576286316, "seconds": 0.29240399699995123, "nonpadding_tokens_per_second": 228827.2413732127, "ranks": [{"compute_seconds": 0.23216247899995324, "nonpadding_tokens": 7795, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.26169589199997745, "nonpadding_tokens": 11733, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.258142423000038, "nonpadding_tokens": 15843, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2588914020000175, "nonpadding_tokens": 2774, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26091118700003335, "nonpadding_tokens": 15404, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.25764375699998254, "nonpadding_tokens": 2514, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2567770189999692, "nonpadding_tokens": 2199, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2573598810000135, "nonpadding_tokens": 8648, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004815450000137389, "prefetch": {"data_wait_seconds": 0.0019184789997694907, "consumed_batches": 202, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 752, "tokens": 67303529, "loss_tokens": 80224, "loss": 3.357033898833267, "lr": 0.00029993989651035607, "optimizer_lrs": [0.019995993100690408, 0.00029993989651035607], "gradient_norm": 0.5986261963844299, "seconds": 0.2535384399999998, "nonpadding_tokens_per_second": 316417.50260828325, "ranks": [{"compute_seconds": 0.25114415899997766, "nonpadding_tokens": 4097, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.24797346399998332, "nonpadding_tokens": 21749, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25058754999997745, "nonpadding_tokens": 18585, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2488843120000297, "nonpadding_tokens": 17073, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24728580900000452, "nonpadding_tokens": 2913, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.24989798799998653, "nonpadding_tokens": 9765, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.25087199899996904, "nonpadding_tokens": 1733, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.24755245099999001, "nonpadding_tokens": 4309, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004461679999963053, "prefetch": {"data_wait_seconds": 0.0019289849997363717, "consumed_batches": 203, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 753, "tokens": 67376791, "loss_tokens": 73262, "loss": 3.4967539447462532, "lr": 0.00029993971020782174, "optimizer_lrs": [0.01999598068052145, 0.00029993971020782174], "gradient_norm": 0.5841196179389954, "seconds": 0.2437750519999895, "nonpadding_tokens_per_second": 300531.16345967655, "ranks": [{"compute_seconds": 0.24145587800001067, "nonpadding_tokens": 1605, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.23795823399996152, "nonpadding_tokens": 3827, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23850939700002982, "nonpadding_tokens": 3957, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23927077700000154, "nonpadding_tokens": 5830, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23756973899998002, "nonpadding_tokens": 9836, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.24066203500001393, "nonpadding_tokens": 20799, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.24152269199998955, "nonpadding_tokens": 24336, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.238367702000005, "nonpadding_tokens": 3072, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036937500004796675, "prefetch": {"data_wait_seconds": 0.0019377039997294787, "consumed_batches": 204, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 754, "tokens": 67430448, "loss_tokens": 53657, "loss": 3.1132701115418304, "lr": 0.0002999395735772177, "optimizer_lrs": [0.019995971571814517, 0.0002999395735772177], "gradient_norm": 0.4929317235946655, "seconds": 0.24648271499995644, "nonpadding_tokens_per_second": 217690.72123377692, "ranks": [{"compute_seconds": 0.24349416600000495, "nonpadding_tokens": 6767, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.24412250000000313, "nonpadding_tokens": 4375, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24108919299999343, "nonpadding_tokens": 7971, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.24360319300001265, "nonpadding_tokens": 6054, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24418773800005056, "nonpadding_tokens": 4270, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.24136960699996735, "nonpadding_tokens": 13705, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.24356075999997984, "nonpadding_tokens": 5251, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.24074966999995695, "nonpadding_tokens": 5264, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003454499999975269, "prefetch": {"data_wait_seconds": 0.001946617999763589, "consumed_batches": 205, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 755, "tokens": 67496335, "loss_tokens": 65887, "loss": 3.2132823053106074, "lr": 0.0002999394055930177, "optimizer_lrs": [0.019995960372867848, 0.0002999394055930177], "gradient_norm": 0.5547484755516052, "seconds": 0.37000005499999133, "nonpadding_tokens_per_second": 178072.94650267428, "ranks": [{"compute_seconds": 0.36579042700003583, "nonpadding_tokens": 12986, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3677294139999958, "nonpadding_tokens": 11499, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3676781390000201, "nonpadding_tokens": 17342, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.36570442099997535, "nonpadding_tokens": 3850, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.36353886700004523, "nonpadding_tokens": 4573, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.36490131499999734, "nonpadding_tokens": 3890, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.36757919699999775, "nonpadding_tokens": 7034, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3644546710000327, "nonpadding_tokens": 4713, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037202300001126787, "prefetch": {"data_wait_seconds": 0.0019549919997672305, "consumed_batches": 206, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 756, "tokens": 67540999, "loss_tokens": 44664, "loss": 3.1253652270284795, "lr": 0.00029993929158598375, "optimizer_lrs": [0.01999595277239892, 0.00029993929158598375], "gradient_norm": 0.5088188052177429, "seconds": 0.2500151520000031, "nonpadding_tokens_per_second": 178645.17267337238, "ranks": [{"compute_seconds": 0.22722850900004232, "nonpadding_tokens": 7046, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.23672586000003548, "nonpadding_tokens": 2690, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23586524399996733, "nonpadding_tokens": 8885, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23388074299998607, "nonpadding_tokens": 2865, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23637790499998346, "nonpadding_tokens": 4016, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.23547044899999037, "nonpadding_tokens": 3557, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.23180607599999803, "nonpadding_tokens": 8896, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.23695008900000403, "nonpadding_tokens": 6709, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0059279120000041985, "prefetch": {"data_wait_seconds": 0.001963402999777486, "consumed_batches": 207, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 757, "tokens": 67589920, "loss_tokens": 48921, "loss": 3.3305887936673413, "lr": 0.000299939166589822, "optimizer_lrs": [0.01999594443932147, 0.000299939166589822], "gradient_norm": 0.7309131026268005, "seconds": 0.37806420499998694, "nonpadding_tokens_per_second": 129398.65597697007, "ranks": [{"compute_seconds": 0.37498632500000895, "nonpadding_tokens": 8520, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3757379970000443, "nonpadding_tokens": 14169, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3743999960000224, "nonpadding_tokens": 4668, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.37470516899998074, "nonpadding_tokens": 1956, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.371268319999956, "nonpadding_tokens": 2936, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.37543469000002005, "nonpadding_tokens": 10191, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.37132658699999865, "nonpadding_tokens": 4641, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.37171556299995245, "nonpadding_tokens": 1840, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003786460000014813, "prefetch": {"data_wait_seconds": 0.0019725979997957666, "consumed_batches": 208, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 758, "tokens": 67620592, "loss_tokens": 30672, "loss": 3.3758010665264737, "lr": 0.0002999390881554278, "optimizer_lrs": [0.019995939210361857, 0.0002999390881554278], "gradient_norm": 0.5468186736106873, "seconds": 0.20992836099998158, "nonpadding_tokens_per_second": 146106.98551589553, "ranks": [{"compute_seconds": 0.20664162100001704, "nonpadding_tokens": 2799, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.20306663899998512, "nonpadding_tokens": 5943, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20375271199998224, "nonpadding_tokens": 3373, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2041763670000023, "nonpadding_tokens": 2071, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.20240044100000887, "nonpadding_tokens": 5251, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.20385106199995562, "nonpadding_tokens": 3549, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2021937019999882, "nonpadding_tokens": 4063, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.20731790999997202, "nonpadding_tokens": 3623, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039034499997114835, "prefetch": {"data_wait_seconds": 0.0019828139998168126, "consumed_batches": 209, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 759, "tokens": 67698548, "loss_tokens": 77956, "loss": 3.2368786158217455, "lr": 0.00029993888857908344, "optimizer_lrs": [0.01999592590527223, 0.00029993888857908344], "gradient_norm": 0.5085577368736267, "seconds": 0.2611992839999857, "nonpadding_tokens_per_second": 298454.1106169505, "ranks": [{"compute_seconds": 0.2563579549999986, "nonpadding_tokens": 6159, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2550159539999868, "nonpadding_tokens": 15255, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2579991090000249, "nonpadding_tokens": 12214, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2581127860000265, "nonpadding_tokens": 6446, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.258858875000044, "nonpadding_tokens": 2250, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.25741516300001877, "nonpadding_tokens": 10991, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.258823499000016, "nonpadding_tokens": 9398, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.25518659499999785, "nonpadding_tokens": 15243, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036818500001345456, "prefetch": {"data_wait_seconds": 0.001991087999797969, "consumed_batches": 210, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 760, "tokens": 67753780, "loss_tokens": 55232, "loss": 3.5490043702020566, "lr": 0.0002999387469812812, "optimizer_lrs": [0.019995916465418748, 0.0002999387469812812], "gradient_norm": 0.49040889739990234, "seconds": 0.23463716199995588, "nonpadding_tokens_per_second": 235393.23238153718, "ranks": [{"compute_seconds": 0.23051792900002965, "nonpadding_tokens": 5233, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.23171043599995755, "nonpadding_tokens": 5325, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22924474199999167, "nonpadding_tokens": 11799, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2294477130000132, "nonpadding_tokens": 4445, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2282040590000065, "nonpadding_tokens": 4668, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.23025260499997557, "nonpadding_tokens": 4241, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.22827844500000083, "nonpadding_tokens": 14439, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.23223652699999775, "nonpadding_tokens": 5082, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004045979999887095, "prefetch": {"data_wait_seconds": 0.0019997709997738866, "consumed_batches": 211, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 761, "tokens": 67830435, "loss_tokens": 76655, "loss": 2.607230448111669, "lr": 0.00029993855019006834, "optimizer_lrs": [0.019995903346004557, 0.00029993855019006834], "gradient_norm": 2.3577687740325928, "seconds": 0.3415193130000489, "nonpadding_tokens_per_second": 224452.8993884133, "ranks": [{"compute_seconds": 0.31032010000001264, "nonpadding_tokens": 13329, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3107956359999662, "nonpadding_tokens": 2329, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3136597630000324, "nonpadding_tokens": 6429, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3107823149999831, "nonpadding_tokens": 1855, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3085662539999703, "nonpadding_tokens": 3777, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.31179770900001813, "nonpadding_tokens": 4051, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3085141779999958, "nonpadding_tokens": 25025, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3153264379999996, "nonpadding_tokens": 19860, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01829162400002815, "prefetch": {"data_wait_seconds": 0.002007499999777451, "consumed_batches": 212, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 762, "tokens": 67997545, "loss_tokens": 167110, "loss": 2.329876092095027, "lr": 0.0002999381200860766, "optimizer_lrs": [0.019995874672405108, 0.0002999381200860766], "gradient_norm": 2.5428273677825928, "seconds": 0.31403166400002647, "nonpadding_tokens_per_second": 532143.7904426921, "ranks": [{"compute_seconds": 0.308587158000023, "nonpadding_tokens": 25193, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.310664810999981, "nonpadding_tokens": 23389, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31099615799996627, "nonpadding_tokens": 18329, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3116209390000222, "nonpadding_tokens": 25891, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30700168800001393, "nonpadding_tokens": 24908, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.30929039099999045, "nonpadding_tokens": 16710, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.30627890099998467, "nonpadding_tokens": 17561, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.30702085099994747, "nonpadding_tokens": 15129, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036935499997525767, "prefetch": {"data_wait_seconds": 0.002016151999796989, "consumed_batches": 213, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 763, "tokens": 68047843, "loss_tokens": 50298, "loss": 3.225958599248479, "lr": 0.00029993799033659475, "optimizer_lrs": [0.019995866022439652, 0.00029993799033659475], "gradient_norm": 0.5359655618667603, "seconds": 0.22862398399996664, "nonpadding_tokens_per_second": 220003.164672379, "ranks": [{"compute_seconds": 0.19223438599999554, "nonpadding_tokens": 4413, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.20039285199999313, "nonpadding_tokens": 21080, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19863548199998604, "nonpadding_tokens": 3972, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.20314454000003934, "nonpadding_tokens": 2072, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.19750815899999452, "nonpadding_tokens": 4394, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2001621, "nonpadding_tokens": 4862, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.19722340399999894, "nonpadding_tokens": 5120, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2041149450000148, "nonpadding_tokens": 4385, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.010538581000048453, "prefetch": {"data_wait_seconds": 0.00202592199981666, "consumed_batches": 214, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 764, "tokens": 68096517, "loss_tokens": 48674, "loss": 3.2155080869663477, "lr": 0.0002999378646470614, "optimizer_lrs": [0.01999585764313743, 0.0002999378646470614], "gradient_norm": 0.48171088099479675, "seconds": 0.24974112600000353, "nonpadding_tokens_per_second": 194897.81590877954, "ranks": [{"compute_seconds": 0.24701970000000983, "nonpadding_tokens": 6184, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.24534977000001845, "nonpadding_tokens": 8392, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24246554900003048, "nonpadding_tokens": 3803, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2431705349999902, "nonpadding_tokens": 7435, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24131180500000937, "nonpadding_tokens": 4904, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.24428543099998024, "nonpadding_tokens": 9578, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.24089635199999293, "nonpadding_tokens": 6131, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2420100679999564, "nonpadding_tokens": 2247, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000388395000015862, "prefetch": {"data_wait_seconds": 0.002035579999827064, "consumed_batches": 215, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 765, "tokens": 68145933, "loss_tokens": 49416, "loss": 3.0472778300550427, "lr": 0.00029993773691133394, "optimizer_lrs": [0.019995849127422263, 0.00029993773691133394], "gradient_norm": 0.5954259634017944, "seconds": 0.33549848600000587, "nonpadding_tokens_per_second": 147291.27570488988, "ranks": [{"compute_seconds": 0.3300233929999763, "nonpadding_tokens": 3339, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.33083909700002323, "nonpadding_tokens": 1690, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33151785399996925, "nonpadding_tokens": 6783, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3286603640000294, "nonpadding_tokens": 3587, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32712592799998674, "nonpadding_tokens": 3844, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3295017980000239, "nonpadding_tokens": 12812, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.32721544899999344, "nonpadding_tokens": 4716, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3329369830000246, "nonpadding_tokens": 12645, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004760699999906137, "prefetch": {"data_wait_seconds": 0.002045395999815014, "consumed_batches": 216, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 766, "tokens": 68202770, "loss_tokens": 56837, "loss": 3.2758560114889947, "lr": 0.00029993758983086716, "optimizer_lrs": [0.019995839322057812, 0.00029993758983086716], "gradient_norm": 0.5187922716140747, "seconds": 0.29269795400000476, "nonpadding_tokens_per_second": 194183.1134221016, "ranks": [{"compute_seconds": 0.2862111059999961, "nonpadding_tokens": 4045, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.28688480300002084, "nonpadding_tokens": 5503, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2882223840000506, "nonpadding_tokens": 10196, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2843598900000188, "nonpadding_tokens": 15929, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28240089499996657, "nonpadding_tokens": 10358, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2869327590000239, "nonpadding_tokens": 3250, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2821983460000297, "nonpadding_tokens": 1939, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.289918431999979, "nonpadding_tokens": 5617, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004020690000174909, "prefetch": {"data_wait_seconds": 0.00205559899980301, "consumed_batches": 217, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 767, "tokens": 68243581, "loss_tokens": 40811, "loss": 3.1860887689593493, "lr": 0.00029993748411482837, "optimizer_lrs": [0.019995832274321893, 0.00029993748411482837], "gradient_norm": 0.5543181896209717, "seconds": 0.27873187900001994, "nonpadding_tokens_per_second": 146416.69315477574, "ranks": [{"compute_seconds": 0.27208497999998826, "nonpadding_tokens": 4437, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2723573700000088, "nonpadding_tokens": 9386, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2754202880000207, "nonpadding_tokens": 3626, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27168379300002243, "nonpadding_tokens": 5280, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2703815860000418, "nonpadding_tokens": 2478, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2731057729999975, "nonpadding_tokens": 7674, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2701313869999922, "nonpadding_tokens": 2477, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2762625009999624, "nonpadding_tokens": 5453, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042790099996636854, "prefetch": {"data_wait_seconds": 0.002065018999829249, "consumed_batches": 218, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 768, "tokens": 68294345, "loss_tokens": 50764, "loss": 3.0778596791033017, "lr": 0.00029993735249190964, "optimizer_lrs": [0.019995823499460646, 0.00029993735249190964], "gradient_norm": 0.5837520360946655, "seconds": 0.3696764360000202, "nonpadding_tokens_per_second": 137320.0860441027, "ranks": [{"compute_seconds": 0.36529741399999693, "nonpadding_tokens": 6709, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3656917619999831, "nonpadding_tokens": 5359, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3620736489999672, "nonpadding_tokens": 2274, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.36266700999999557, "nonpadding_tokens": 5837, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3664902589999883, "nonpadding_tokens": 11960, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.36370348599996305, "nonpadding_tokens": 2370, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3610803689999784, "nonpadding_tokens": 5046, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.36718636699998797, "nonpadding_tokens": 11209, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003619780000008177, "prefetch": {"data_wait_seconds": 0.0020731669998355073, "consumed_batches": 219, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 769, "tokens": 68378461, "loss_tokens": 84116, "loss": 3.31777025179514, "lr": 0.00029993713408797094, "optimizer_lrs": [0.019995808939198063, 0.00029993713408797094], "gradient_norm": 0.45969924330711365, "seconds": 0.29637214299998504, "nonpadding_tokens_per_second": 283818.84730645636, "ranks": [{"compute_seconds": 0.2488736750000271, "nonpadding_tokens": 11745, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2633330479999927, "nonpadding_tokens": 12332, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26204116299999214, "nonpadding_tokens": 11431, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2664055299999859, "nonpadding_tokens": 15000, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26719685400001936, "nonpadding_tokens": 6482, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.26350735400001213, "nonpadding_tokens": 3766, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2605395459999613, "nonpadding_tokens": 13310, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2672569989999829, "nonpadding_tokens": 10050, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.022701403000041864, "prefetch": {"data_wait_seconds": 0.0020835339998512836, "consumed_batches": 220, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 770, "tokens": 68457414, "loss_tokens": 78953, "loss": 3.4921353526781758, "lr": 0.0002999369287438753, "optimizer_lrs": [0.019995795249591688, 0.0002999369287438753], "gradient_norm": 0.530616819858551, "seconds": 0.4703169439999897, "nonpadding_tokens_per_second": 167871.9021443585, "ranks": [{"compute_seconds": 0.4680045250000262, "nonpadding_tokens": 9567, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.46290819899996904, "nonpadding_tokens": 11178, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.46586086799999293, "nonpadding_tokens": 11448, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.4625517119999927, "nonpadding_tokens": 11282, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.46124197699998604, "nonpadding_tokens": 13624, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.4641044790000137, "nonpadding_tokens": 6779, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.46072611800002505, "nonpadding_tokens": 5238, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.46749078399994914, "nonpadding_tokens": 9837, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003326430000356595, "prefetch": {"data_wait_seconds": 0.002091619999831096, "consumed_batches": 221, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 771, "tokens": 68532078, "loss_tokens": 74664, "loss": 2.798673683100289, "lr": 0.0002999367342468224, "optimizer_lrs": [0.019995782283121495, 0.0002999367342468224], "gradient_norm": 0.5457352995872498, "seconds": 0.2472784820000129, "nonpadding_tokens_per_second": 301942.9729433397, "ranks": [{"compute_seconds": 0.24171446799999785, "nonpadding_tokens": 2645, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2383587539999894, "nonpadding_tokens": 8830, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23873473700001568, "nonpadding_tokens": 9408, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23941340300001457, "nonpadding_tokens": 14889, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24477512299995396, "nonpadding_tokens": 17630, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.24070286800002805, "nonpadding_tokens": 7322, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.24199655800003939, "nonpadding_tokens": 5430, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.23873237099996913, "nonpadding_tokens": 8510, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003886720000423338, "prefetch": {"data_wait_seconds": 0.002101102999816362, "consumed_batches": 222, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 772, "tokens": 68621042, "loss_tokens": 88964, "loss": 2.9870333786700236, "lr": 0.00029993650210792755, "optimizer_lrs": [0.01999576680719517, 0.00029993650210792755], "gradient_norm": 0.5296372175216675, "seconds": 0.3842300629999613, "nonpadding_tokens_per_second": 231538.36351428065, "ranks": [{"compute_seconds": 0.3804149029999735, "nonpadding_tokens": 14029, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3817624150000256, "nonpadding_tokens": 20695, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37855529300003354, "nonpadding_tokens": 4263, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.38179105400001845, "nonpadding_tokens": 10933, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.37743930999999975, "nonpadding_tokens": 14956, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.38005199600002015, "nonpadding_tokens": 9296, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.37773346299997, "nonpadding_tokens": 11005, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.37757884199999125, "nonpadding_tokens": 3787, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005027199999858567, "prefetch": {"data_wait_seconds": 0.0021098569997803907, "consumed_batches": 223, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 773, "tokens": 68668157, "loss_tokens": 47115, "loss": 3.46580905762496, "lr": 0.0002999363789958978, "optimizer_lrs": [0.01999575859972652, 0.0002999363789958978], "gradient_norm": 0.5747690796852112, "seconds": 0.21630579899999702, "nonpadding_tokens_per_second": 217816.62913253956, "ranks": [{"compute_seconds": 0.21398454999996375, "nonpadding_tokens": 2978, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.21101028399999677, "nonpadding_tokens": 5302, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21063465399998904, "nonpadding_tokens": 3664, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2114835630000016, "nonpadding_tokens": 12001, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.20989948800001912, "nonpadding_tokens": 9730, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.21140248599999723, "nonpadding_tokens": 3556, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2092466189999982, "nonpadding_tokens": 3388, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.21001912600002015, "nonpadding_tokens": 6496, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035545399998682115, "prefetch": {"data_wait_seconds": 0.0021195289998132694, "consumed_batches": 224, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 774, "tokens": 68809797, "loss_tokens": 141640, "loss": 2.7169050939000283, "lr": 0.00029993600817113586, "optimizer_lrs": [0.019995733878075725, 0.00029993600817113586], "gradient_norm": 1.090044379234314, "seconds": 0.2623926429999983, "nonpadding_tokens_per_second": 539801.7199742941, "ranks": [{"compute_seconds": 0.23485916100003124, "nonpadding_tokens": 7191, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.23483407200001238, "nonpadding_tokens": 4698, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23735306800000444, "nonpadding_tokens": 14267, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23350425499995708, "nonpadding_tokens": 12298, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.231707694000022, "nonpadding_tokens": 24891, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.23486731899998858, "nonpadding_tokens": 23274, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.23142094199999974, "nonpadding_tokens": 27657, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.23274789800001372, "nonpadding_tokens": 27364, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.016665489999979854, "prefetch": {"data_wait_seconds": 0.0021292859997998903, "consumed_batches": 225, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 775, "tokens": 68896055, "loss_tokens": 86258, "loss": 2.397960370342461, "lr": 0.00029993578181303383, "optimizer_lrs": [0.01999571878753559, 0.00029993578181303383], "gradient_norm": 1.3866089582443237, "seconds": 0.3503264899999863, "nonpadding_tokens_per_second": 246221.74589196316, "ranks": [{"compute_seconds": 0.34803217499995753, "nonpadding_tokens": 17213, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3479333729999894, "nonpadding_tokens": 11002, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34456433100001504, "nonpadding_tokens": 22104, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3451922460000105, "nonpadding_tokens": 8732, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34787117199999784, "nonpadding_tokens": 5722, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.34662726099998054, "nonpadding_tokens": 5648, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.343960385999992, "nonpadding_tokens": 12123, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3444464309999944, "nonpadding_tokens": 3714, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040103899999621717, "prefetch": {"data_wait_seconds": 0.0021370789998087503, "consumed_batches": 226, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 776, "tokens": 68952518, "loss_tokens": 56463, "loss": 3.17460494040345, "lr": 0.00029993563342655544, "optimizer_lrs": [0.019995708895103697, 0.00029993563342655544], "gradient_norm": 0.6073852181434631, "seconds": 0.298909277000007, "nonpadding_tokens_per_second": 188896.7802093298, "ranks": [{"compute_seconds": 0.29238465200000974, "nonpadding_tokens": 3191, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2910768810000377, "nonpadding_tokens": 12778, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2902081259999818, "nonpadding_tokens": 15426, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2943969739999943, "nonpadding_tokens": 2795, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28983470700001135, "nonpadding_tokens": 5162, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.292312064999976, "nonpadding_tokens": 3058, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2945936090000032, "nonpadding_tokens": 10299, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.29651046200001474, "nonpadding_tokens": 3754, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039378000002443514, "prefetch": {"data_wait_seconds": 0.002145089999842753, "consumed_batches": 227, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 777, "tokens": 69002877, "loss_tokens": 50359, "loss": 3.263466435989595, "lr": 0.0002999355009371384, "optimizer_lrs": [0.019995700062475894, 0.0002999355009371384], "gradient_norm": 0.5420112609863281, "seconds": 0.23338120600004686, "nonpadding_tokens_per_second": 215780.01443693752, "ranks": [{"compute_seconds": 0.2289172930000518, "nonpadding_tokens": 2608, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2273098879999793, "nonpadding_tokens": 6382, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22737785699996493, "nonpadding_tokens": 3028, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23045352100001537, "nonpadding_tokens": 8583, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.22706497599995146, "nonpadding_tokens": 7512, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.23001764200000707, "nonpadding_tokens": 5739, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.23100164400000267, "nonpadding_tokens": 4400, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2274936399999774, "nonpadding_tokens": 12107, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003713550000270516, "prefetch": {"data_wait_seconds": 0.0021549689998465738, "consumed_batches": 228, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 778, "tokens": 69053953, "loss_tokens": 51076, "loss": 2.978293070620252, "lr": 0.00029993536642226225, "optimizer_lrs": [0.019995691094817483, 0.00029993536642226225], "gradient_norm": 0.5966860055923462, "seconds": 0.30385259799999176, "nonpadding_tokens_per_second": 168094.66279436383, "ranks": [{"compute_seconds": 0.3016442190000248, "nonpadding_tokens": 3862, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.29748124900004314, "nonpadding_tokens": 6344, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29799451399998134, "nonpadding_tokens": 3772, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.29830246699998497, "nonpadding_tokens": 7821, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3011085760000469, "nonpadding_tokens": 9829, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.29984555599997975, "nonpadding_tokens": 10454, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.29699692900004493, "nonpadding_tokens": 3653, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2972884330000056, "nonpadding_tokens": 5341, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038647700000637997, "prefetch": {"data_wait_seconds": 0.0021633679998700472, "consumed_batches": 229, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 779, "tokens": 69103678, "loss_tokens": 49725, "loss": 2.991139077425842, "lr": 0.0002999352353308307, "optimizer_lrs": [0.019995682355388716, 0.0002999352353308307], "gradient_norm": 0.5411214828491211, "seconds": 0.28386961800003974, "nonpadding_tokens_per_second": 175168.44652249131, "ranks": [{"compute_seconds": 0.26185603000004676, "nonpadding_tokens": 10684, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.27188860600000453, "nonpadding_tokens": 3202, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27138964300002044, "nonpadding_tokens": 5410, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2722093490000361, "nonpadding_tokens": 6212, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2723682609999969, "nonpadding_tokens": 3246, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27060835599996835, "nonpadding_tokens": 5639, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2674427439999931, "nonpadding_tokens": 5836, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2735935200000199, "nonpadding_tokens": 9496, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00581221900000628, "prefetch": {"data_wait_seconds": 0.0021710999998418856, "consumed_batches": 230, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 780, "tokens": 69190337, "loss_tokens": 86659, "loss": 3.3406159487185403, "lr": 0.000299935006551913, "optimizer_lrs": [0.01999566710346087, 0.000299935006551913], "gradient_norm": 0.45000937581062317, "seconds": 0.35020484400001806, "nonpadding_tokens_per_second": 247452.31679318385, "ranks": [{"compute_seconds": 0.3438995979999504, "nonpadding_tokens": 20496, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.34311907299996847, "nonpadding_tokens": 6039, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3463483079999605, "nonpadding_tokens": 15974, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.34650940199998104, "nonpadding_tokens": 15648, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34137451499998406, "nonpadding_tokens": 10624, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3440300390000175, "nonpadding_tokens": 11831, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3416951240000117, "nonpadding_tokens": 3892, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3477917760000082, "nonpadding_tokens": 2155, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036543499999197593, "prefetch": {"data_wait_seconds": 0.0021792809998402163, "consumed_batches": 231, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 781, "tokens": 69250598, "loss_tokens": 60261, "loss": 3.377186841406548, "lr": 0.0002999348472257456, "optimizer_lrs": [0.019995656481716372, 0.0002999348472257456], "gradient_norm": 0.4508289098739624, "seconds": 0.29666380599996955, "nonpadding_tokens_per_second": 203128.9250027561, "ranks": [{"compute_seconds": 0.26257183600000644, "nonpadding_tokens": 4785, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2825810069999761, "nonpadding_tokens": 20685, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2802355540000008, "nonpadding_tokens": 5100, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2801673870000059, "nonpadding_tokens": 7786, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27876131599998644, "nonpadding_tokens": 2505, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.28106602500002964, "nonpadding_tokens": 8145, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.28332214499999964, "nonpadding_tokens": 7133, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2792140169999584, "nonpadding_tokens": 4122, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004670170000053986, "prefetch": {"data_wait_seconds": 0.002187302999814733, "consumed_batches": 232, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 782, "tokens": 69321913, "loss_tokens": 71315, "loss": 3.1434596946645166, "lr": 0.0002999346584215833, "optimizer_lrs": [0.01999564389477222, 0.0002999346584215833], "gradient_norm": 0.6732993721961975, "seconds": 0.29955365400002165, "nonpadding_tokens_per_second": 238070.873273323, "ranks": [{"compute_seconds": 0.2971811620000153, "nonpadding_tokens": 7248, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2931539939999652, "nonpadding_tokens": 13449, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29162612599998283, "nonpadding_tokens": 9977, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2924579030000132, "nonpadding_tokens": 9048, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29075813700001163, "nonpadding_tokens": 7740, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2937566179999749, "nonpadding_tokens": 4902, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2910631549999607, "nonpadding_tokens": 14017, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2911075949999713, "nonpadding_tokens": 4934, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039639299995997135, "prefetch": {"data_wait_seconds": 0.0021954389998342094, "consumed_batches": 233, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 783, "tokens": 69365947, "loss_tokens": 44034, "loss": 3.356128573942862, "lr": 0.0002999345417066129, "optimizer_lrs": [0.019995636113774196, 0.0002999345417066129], "gradient_norm": 1.0597362518310547, "seconds": 0.32780061399995475, "nonpadding_tokens_per_second": 134331.65808532035, "ranks": [{"compute_seconds": 0.3245485950000102, "nonpadding_tokens": 2826, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.32236611299998685, "nonpadding_tokens": 11305, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3208684909999988, "nonpadding_tokens": 5548, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3253067750000014, "nonpadding_tokens": 4620, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3204289820000099, "nonpadding_tokens": 7014, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3228836960000194, "nonpadding_tokens": 3635, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.32539139699997577, "nonpadding_tokens": 2916, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3206334529999708, "nonpadding_tokens": 6170, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003723159999822201, "prefetch": {"data_wait_seconds": 0.00220442599983528, "consumed_batches": 234, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 784, "tokens": 69426664, "loss_tokens": 60717, "loss": 3.1047127451949206, "lr": 0.00029993438060150357, "optimizer_lrs": [0.019995625373433572, 0.00029993438060150357], "gradient_norm": 0.5297110080718994, "seconds": 0.4386181479999891, "nonpadding_tokens_per_second": 138427.9247834531, "ranks": [{"compute_seconds": 0.4356102859999851, "nonpadding_tokens": 18428, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.4317108799999687, "nonpadding_tokens": 3587, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4321027379999691, "nonpadding_tokens": 6574, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.43249586600001066, "nonpadding_tokens": 4324, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4356992269999864, "nonpadding_tokens": 1628, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.43329494399995383, "nonpadding_tokens": 9319, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.43097253600001295, "nonpadding_tokens": 9875, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.43622066400001813, "nonpadding_tokens": 6982, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034472400000140624, "prefetch": {"data_wait_seconds": 0.0022127929998561058, "consumed_batches": 235, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 785, "tokens": 69502060, "loss_tokens": 75396, "loss": 3.2986220257042813, "lr": 0.0002999341802719308, "optimizer_lrs": [0.019995612018128724, 0.0002999341802719308], "gradient_norm": 0.8102033734321594, "seconds": 0.5492817610000316, "nonpadding_tokens_per_second": 137262.8864696559, "ranks": [{"compute_seconds": 0.5462028480000072, "nonpadding_tokens": 13520, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.5435391329999675, "nonpadding_tokens": 16969, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5440425980000327, "nonpadding_tokens": 7913, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.5447683649999817, "nonpadding_tokens": 4427, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.5469242009999675, "nonpadding_tokens": 13346, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.544496636999952, "nonpadding_tokens": 2102, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.5425100719999705, "nonpadding_tokens": 7840, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.5435779399999774, "nonpadding_tokens": 9279, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037458899998910056, "prefetch": {"data_wait_seconds": 0.0022301449998423095, "consumed_batches": 236, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 786, "tokens": 69579624, "loss_tokens": 77564, "loss": 3.2138552517920687, "lr": 0.0002999339738633678, "optimizer_lrs": [0.019995598257557857, 0.0002999339738633678], "gradient_norm": 0.5087431073188782, "seconds": 0.31889673300003096, "nonpadding_tokens_per_second": 243226.07281145293, "ranks": [{"compute_seconds": 0.3111050480000017, "nonpadding_tokens": 27188, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.313719591999984, "nonpadding_tokens": 12494, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31458074500000066, "nonpadding_tokens": 6235, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3153474989999836, "nonpadding_tokens": 2414, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3093446569999969, "nonpadding_tokens": 3388, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3132149639999966, "nonpadding_tokens": 13472, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3093753290000336, "nonpadding_tokens": 4068, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.31627799900002174, "nonpadding_tokens": 8305, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005147379999925761, "prefetch": {"data_wait_seconds": 0.0022385109998737107, "consumed_batches": 237, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 787, "tokens": 69670019, "loss_tokens": 90395, "loss": 2.8921267423530064, "lr": 0.00029993373290209096, "optimizer_lrs": [0.019995582193472732, 0.00029993373290209096], "gradient_norm": 0.5924637317657471, "seconds": 0.3077214410000124, "nonpadding_tokens_per_second": 293755.9362332388, "ranks": [{"compute_seconds": 0.3020281069999555, "nonpadding_tokens": 10093, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.30152248799998915, "nonpadding_tokens": 8217, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30425789299999906, "nonpadding_tokens": 7836, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3013543860000141, "nonpadding_tokens": 17315, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3050166509999599, "nonpadding_tokens": 19274, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.30210309199998164, "nonpadding_tokens": 4471, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2996912539999812, "nonpadding_tokens": 10377, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.30524447400000554, "nonpadding_tokens": 12812, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039020299999492636, "prefetch": {"data_wait_seconds": 0.0022473209998707944, "consumed_batches": 238, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 788, "tokens": 69798046, "loss_tokens": 128027, "loss": 2.9094402450264396, "lr": 0.00029993339087632887, "optimizer_lrs": [0.01999555939175526, 0.00029993339087632887], "gradient_norm": 0.48893627524375916, "seconds": 0.282446297999968, "nonpadding_tokens_per_second": 453279.0867027562, "ranks": [{"compute_seconds": 0.27789946600000803, "nonpadding_tokens": 4003, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.27479386400000294, "nonpadding_tokens": 15980, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27461114300001555, "nonpadding_tokens": 21339, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2780398699999864, "nonpadding_tokens": 16716, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2783156559999611, "nonpadding_tokens": 12252, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27628726799997594, "nonpadding_tokens": 6943, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.27328938300001937, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2798212110000122, "nonpadding_tokens": 18026, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003757300000302166, "prefetch": {"data_wait_seconds": 0.0022554139998902656, "consumed_batches": 239, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 789, "tokens": 69922988, "loss_tokens": 124942, "loss": 2.828548446479166, "lr": 0.0002999330562435773, "optimizer_lrs": [0.019995537082905156, 0.0002999330562435773], "gradient_norm": 2.042248487472534, "seconds": 0.32349259000000075, "nonpadding_tokens_per_second": 386228.32133496384, "ranks": [{"compute_seconds": 0.3204644620000181, "nonpadding_tokens": 6512, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.31844088899998724, "nonpadding_tokens": 8485, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31577610999994477, "nonpadding_tokens": 11928, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.32018694500004585, "nonpadding_tokens": 30789, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3151387720000116, "nonpadding_tokens": 20235, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3178886489999968, "nonpadding_tokens": 10675, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3144560819999924, "nonpadding_tokens": 23682, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3211252560000162, "nonpadding_tokens": 12636, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039048000002139815, "prefetch": {"data_wait_seconds": 0.002263526999868759, "consumed_batches": 240, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 790, "tokens": 70042238, "loss_tokens": 119250, "loss": 2.8114119496855348, "lr": 0.00029993273607389377, "optimizer_lrs": [0.019995515738259587, 0.00029993273607389377], "gradient_norm": 1.6603538990020752, "seconds": 0.37370006500003683, "nonpadding_tokens_per_second": 319106.1794436355, "ranks": [{"compute_seconds": 0.37002951099998427, "nonpadding_tokens": 19820, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.36708172700002706, "nonpadding_tokens": 26305, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3695507769999722, "nonpadding_tokens": 11243, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.36720083500000555, "nonpadding_tokens": 15716, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.36617318100002194, "nonpadding_tokens": 12917, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3681280220000076, "nonpadding_tokens": 17924, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3704044880000197, "nonpadding_tokens": 11878, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.37117843700002595, "nonpadding_tokens": 3447, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004217270000026474, "prefetch": {"data_wait_seconds": 0.0022730139998685672, "consumed_batches": 241, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 791, "tokens": 70122085, "loss_tokens": 79847, "loss": 3.0177949703808533, "lr": 0.0002999325212689644, "optimizer_lrs": [0.01999550141793096, 0.0002999325212689644], "gradient_norm": 0.8813191056251526, "seconds": 0.35119649699998945, "nonpadding_tokens_per_second": 227357.05134326097, "ranks": [{"compute_seconds": 0.3469303429999968, "nonpadding_tokens": 11265, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.348248271999978, "nonpadding_tokens": 13314, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.347863464999989, "nonpadding_tokens": 4627, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3479018279999764, "nonpadding_tokens": 12810, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3445045799999775, "nonpadding_tokens": 7148, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.34572262900002215, "nonpadding_tokens": 4280, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3446476969999708, "nonpadding_tokens": 14976, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3488245099999858, "nonpadding_tokens": 11427, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003760499999998501, "prefetch": {"data_wait_seconds": 0.0022821609998828762, "consumed_batches": 242, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 792, "tokens": 70187816, "loss_tokens": 65731, "loss": 3.1140077550927265, "lr": 0.0002999323441820744, "optimizer_lrs": [0.019995489612138293, 0.0002999323441820744], "gradient_norm": 1.2911087274551392, "seconds": 0.3356590270000197, "nonpadding_tokens_per_second": 195826.70124345008, "ranks": [{"compute_seconds": 0.33101394299995945, "nonpadding_tokens": 13237, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.33031246699999883, "nonpadding_tokens": 11939, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3322406210000395, "nonpadding_tokens": 4752, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3274383680000028, "nonpadding_tokens": 5563, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3260471600000301, "nonpadding_tokens": 7245, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3291300400000523, "nonpadding_tokens": 7293, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3323562389999779, "nonpadding_tokens": 1541, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.33306298100001186, "nonpadding_tokens": 14161, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038595299997723487, "prefetch": {"data_wait_seconds": 0.0022898409998362013, "consumed_batches": 243, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 793, "tokens": 70239202, "loss_tokens": 51386, "loss": 3.155399208928502, "lr": 0.00029993220558068614, "optimizer_lrs": [0.019995480372045744, 0.00029993220558068614], "gradient_norm": 0.6047708988189697, "seconds": 0.2752919509999856, "nonpadding_tokens_per_second": 186660.0160787218, "ranks": [{"compute_seconds": 0.27235164500001474, "nonpadding_tokens": 4453, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2711626149999802, "nonpadding_tokens": 6912, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26834489999998823, "nonpadding_tokens": 4674, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27246429399997396, "nonpadding_tokens": 6618, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2729199149999886, "nonpadding_tokens": 2011, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27050526100003935, "nonpadding_tokens": 3150, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.27265565599998354, "nonpadding_tokens": 10990, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2679580930000043, "nonpadding_tokens": 12578, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004077560000155245, "prefetch": {"data_wait_seconds": 0.002301250999849458, "consumed_batches": 244, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 794, "tokens": 70310074, "loss_tokens": 70872, "loss": 2.87355351020149, "lr": 0.000299932014187858, "optimizer_lrs": [0.019995467612523868, 0.000299932014187858], "gradient_norm": 1.6316159963607788, "seconds": 0.27914484900003345, "nonpadding_tokens_per_second": 253889.69294572764, "ranks": [{"compute_seconds": 0.27299119099996005, "nonpadding_tokens": 4852, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.2728350649999811, "nonpadding_tokens": 11858, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27068141999995987, "nonpadding_tokens": 9694, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27634710500001347, "nonpadding_tokens": 6262, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2694966179999483, "nonpadding_tokens": 5841, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27426602100001674, "nonpadding_tokens": 9638, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.27642687500002694, "nonpadding_tokens": 6543, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.270933991999982, "nonpadding_tokens": 16184, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034689800003206983, "prefetch": {"data_wait_seconds": 0.0023107269998519087, "consumed_batches": 245, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 795, "tokens": 70370862, "loss_tokens": 60788, "loss": 3.017012506580246, "lr": 0.00029993184981241675, "optimizer_lrs": [0.01999545665416112, 0.00029993184981241675], "gradient_norm": 0.5710793137550354, "seconds": 0.3110513689999834, "nonpadding_tokens_per_second": 195427.52759915756, "ranks": [{"compute_seconds": 0.30364404499999864, "nonpadding_tokens": 6200, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3042974620000223, "nonpadding_tokens": 4630, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3072566429999597, "nonpadding_tokens": 3686, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.30320400600004405, "nonpadding_tokens": 8583, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3021714490000136, "nonpadding_tokens": 9561, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3051603689999638, "nonpadding_tokens": 8968, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.307972264, "nonpadding_tokens": 5137, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.30859563000001344, "nonpadding_tokens": 14023, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004414179999798762, "prefetch": {"data_wait_seconds": 0.002319721999882063, "consumed_batches": 246, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 796, "tokens": 70453031, "loss_tokens": 82169, "loss": 3.3165016612104323, "lr": 0.00029993162730571857, "optimizer_lrs": [0.019995441820381238, 0.00029993162730571857], "gradient_norm": 0.5650171637535095, "seconds": 0.3328792009999688, "nonpadding_tokens_per_second": 246843.29856946424, "ranks": [{"compute_seconds": 0.3287397779999992, "nonpadding_tokens": 2527, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.32603559199998244, "nonpadding_tokens": 21578, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3267771630000311, "nonpadding_tokens": 22451, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3291863960000114, "nonpadding_tokens": 7288, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32529731500000025, "nonpadding_tokens": 11033, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.32839951700003667, "nonpadding_tokens": 9679, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.32968264500004807, "nonpadding_tokens": 2344, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.33039299500001107, "nonpadding_tokens": 5269, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045473000000129105, "prefetch": {"data_wait_seconds": 0.002329277999891133, "consumed_batches": 247, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 797, "tokens": 70511194, "loss_tokens": 58163, "loss": 3.3440164924436497, "lr": 0.0002999314695860934, "optimizer_lrs": [0.01999543130573956, 0.0002999314695860934], "gradient_norm": 0.4966455399990082, "seconds": 0.34239312700003666, "nonpadding_tokens_per_second": 169871.9846090654, "ranks": [{"compute_seconds": 0.33881322000002, "nonpadding_tokens": 14237, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18310234112}, {"compute_seconds": 0.3381916789999764, "nonpadding_tokens": 3017, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3385463790000358, "nonpadding_tokens": 4700, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3393859710000129, "nonpadding_tokens": 4595, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.339987807, "nonpadding_tokens": 4824, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3364745699999503, "nonpadding_tokens": 10941, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.33392729100000906, "nonpadding_tokens": 7974, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.33505510599997024, "nonpadding_tokens": 7875, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043671400004541283, "prefetch": {"data_wait_seconds": 0.00233820499990145, "consumed_batches": 248, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 798, "tokens": 70561430, "loss_tokens": 50236, "loss": 2.8270690466000477, "lr": 0.00029993133321577455, "optimizer_lrs": [0.01999542221438497, 0.00029993133321577455], "gradient_norm": 0.6934630274772644, "seconds": 0.3619595629999708, "nonpadding_tokens_per_second": 138788.98400594006, "ranks": [{"compute_seconds": 0.3563954489999901, "nonpadding_tokens": 3136, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35663508200002525, "nonpadding_tokens": 2348, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35826480100001845, "nonpadding_tokens": 12165, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3544196440000178, "nonpadding_tokens": 10415, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3525298540000108, "nonpadding_tokens": 8732, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3556979939999678, "nonpadding_tokens": 5371, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3591302160000396, "nonpadding_tokens": 2987, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3595414100000198, "nonpadding_tokens": 5082, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039255500001900145, "prefetch": {"data_wait_seconds": 0.0023478989998722, "consumed_batches": 249, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 799, "tokens": 70623174, "loss_tokens": 61744, "loss": 3.6727865258162735, "lr": 0.000299931165420279, "optimizer_lrs": [0.019995411028018602, 0.000299931165420279], "gradient_norm": 0.5165947079658508, "seconds": 0.22146848599999203, "nonpadding_tokens_per_second": 278793.6158104328, "ranks": [{"compute_seconds": 0.21727196900002355, "nonpadding_tokens": 4412, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21456644599999208, "nonpadding_tokens": 13902, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21459515899999815, "nonpadding_tokens": 8989, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.21825641099997029, "nonpadding_tokens": 10878, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21364130699998896, "nonpadding_tokens": 11320, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.216833371000007, "nonpadding_tokens": 4490, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.21852101000001767, "nonpadding_tokens": 3548, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2189385419999894, "nonpadding_tokens": 4205, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046756099999356593, "prefetch": {"data_wait_seconds": 0.0023586709998681954, "consumed_batches": 250, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 800, "tokens": 70707485, "loss_tokens": 84311, "loss": 2.943307367959104, "lr": 0.0002999309359660879, "optimizer_lrs": [0.019995395731072528, 0.0002999309359660879], "gradient_norm": 0.5748737454414368, "seconds": 0.26563421500003415, "nonpadding_tokens_per_second": 317395.1066506593, "ranks": [{"compute_seconds": 0.2630279940000264, "nonpadding_tokens": 10470, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2617173730000104, "nonpadding_tokens": 8121, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25860450300001503, "nonpadding_tokens": 4396, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2623579759999757, "nonpadding_tokens": 12033, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25795713499996964, "nonpadding_tokens": 17320, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2603661640000041, "nonpadding_tokens": 10687, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.25813065699998106, "nonpadding_tokens": 13432, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2632507980000014, "nonpadding_tokens": 7852, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036537199997610514, "prefetch": {"data_wait_seconds": 0.0023674429998692403, "consumed_batches": 251, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 801, "tokens": 70753657, "loss_tokens": 46172, "loss": 3.2701826864766526, "lr": 0.00029993081014624746, "optimizer_lrs": [0.019995387343083166, 0.00029993081014624746], "gradient_norm": 0.7927801012992859, "seconds": 0.29575786399999515, "nonpadding_tokens_per_second": 156114.19211494154, "ranks": [{"compute_seconds": 0.29254654599998275, "nonpadding_tokens": 7818, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28955532599997014, "nonpadding_tokens": 3176, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2892155819999971, "nonpadding_tokens": 4587, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2903858459999924, "nonpadding_tokens": 10788, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2883964449999894, "nonpadding_tokens": 3585, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.29134457499998234, "nonpadding_tokens": 3089, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2883915219999835, "nonpadding_tokens": 9705, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.29339654899996503, "nonpadding_tokens": 3424, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004136310000149024, "prefetch": {"data_wait_seconds": 0.0023769799999087127, "consumed_batches": 252, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 802, "tokens": 70810922, "loss_tokens": 57265, "loss": 3.357918231031171, "lr": 0.000299930653938675, "optimizer_lrs": [0.019995376929245, 0.000299930653938675], "gradient_norm": 0.5795595049858093, "seconds": 0.27989999899995155, "nonpadding_tokens_per_second": 204590.9260614535, "ranks": [{"compute_seconds": 0.27731484100002035, "nonpadding_tokens": 3644, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.276981931000023, "nonpadding_tokens": 6845, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2766360440000426, "nonpadding_tokens": 5773, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27697806800000535, "nonpadding_tokens": 5023, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2774883799999657, "nonpadding_tokens": 3435, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27579132899995784, "nonpadding_tokens": 15318, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2772420390000434, "nonpadding_tokens": 12224, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.27753469800001085, "nonpadding_tokens": 5003, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036965300000701973, "prefetch": {"data_wait_seconds": 0.0023855039998608163, "consumed_batches": 253, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 803, "tokens": 70871603, "loss_tokens": 60681, "loss": 3.171017544618579, "lr": 0.0002999304882207739, "optimizer_lrs": [0.019995365881384928, 0.0002999304882207739], "gradient_norm": 0.5317993760108948, "seconds": 0.22562664999998105, "nonpadding_tokens_per_second": 268944.2935930002, "ranks": [{"compute_seconds": 0.22328275099999928, "nonpadding_tokens": 4125, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21953851600000007, "nonpadding_tokens": 4594, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2199365870000065, "nonpadding_tokens": 5046, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.22230323000002272, "nonpadding_tokens": 6267, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2229380679999622, "nonpadding_tokens": 14421, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.22040976599998885, "nonpadding_tokens": 12642, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2188655730000164, "nonpadding_tokens": 1669, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.218897075999962, "nonpadding_tokens": 11917, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003819150000481386, "prefetch": {"data_wait_seconds": 0.0023943009998816933, "consumed_batches": 254, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 804, "tokens": 70926917, "loss_tokens": 55314, "loss": 3.306285479263839, "lr": 0.0002999303369877164, "optimizer_lrs": [0.019995355799181092, 0.0002999303369877164], "gradient_norm": 0.4877786338329315, "seconds": 0.2848633069999664, "nonpadding_tokens_per_second": 194177.34274918924, "ranks": [{"compute_seconds": 0.2814636840000162, "nonpadding_tokens": 4233, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27817572099996823, "nonpadding_tokens": 4108, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2807105499999807, "nonpadding_tokens": 1769, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2811128270000154, "nonpadding_tokens": 6677, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2815350309999758, "nonpadding_tokens": 13192, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2826000689999546, "nonpadding_tokens": 16254, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2795565649999503, "nonpadding_tokens": 6306, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.28004267000000027, "nonpadding_tokens": 2775, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036730300001863725, "prefetch": {"data_wait_seconds": 0.0024031529998893575, "consumed_batches": 255, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 805, "tokens": 71000843, "loss_tokens": 73926, "loss": 3.3229494274680085, "lr": 0.00029993013461139317, "optimizer_lrs": [0.019995342307426212, 0.00029993013461139317], "gradient_norm": 0.5178476572036743, "seconds": 0.2593739999999798, "nonpadding_tokens_per_second": 285017.0024752125, "ranks": [{"compute_seconds": 0.2535728739999854, "nonpadding_tokens": 6225, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25594345999996904, "nonpadding_tokens": 19773, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2530349529999967, "nonpadding_tokens": 11065, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2536180150000291, "nonpadding_tokens": 8119, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25702333699996416, "nonpadding_tokens": 9508, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.25423725299998523, "nonpadding_tokens": 3163, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2515385380000339, "nonpadding_tokens": 4000, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.25220341799996504, "nonpadding_tokens": 12073, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036287299997184164, "prefetch": {"data_wait_seconds": 0.0024112399998585943, "consumed_batches": 256, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 806, "tokens": 71061665, "loss_tokens": 60822, "loss": 3.0566941238367695, "lr": 0.0002999299678879202, "optimizer_lrs": [0.019995331192528016, 0.0002999299678879202], "gradient_norm": 0.9250947833061218, "seconds": 0.2984218020000071, "nonpadding_tokens_per_second": 203812.18661764715, "ranks": [{"compute_seconds": 0.295296153000038, "nonpadding_tokens": 9506, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29198543500001506, "nonpadding_tokens": 5900, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29467509599999175, "nonpadding_tokens": 1761, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.29194076499999255, "nonpadding_tokens": 17922, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29049157899999045, "nonpadding_tokens": 11953, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2927772470000036, "nonpadding_tokens": 6204, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2906036339999787, "nonpadding_tokens": 3242, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.29608497200001693, "nonpadding_tokens": 4334, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036414899994952066, "prefetch": {"data_wait_seconds": 0.0024195559998503313, "consumed_batches": 257, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 807, "tokens": 71104677, "loss_tokens": 43012, "loss": 3.749445648888682, "lr": 0.0002999298498647859, "optimizer_lrs": [0.019995323324319064, 0.0002999298498647859], "gradient_norm": 0.8593497276306152, "seconds": 0.26851544199996624, "nonpadding_tokens_per_second": 160184.45598374715, "ranks": [{"compute_seconds": 0.26417552100002695, "nonpadding_tokens": 1621, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26432269800000086, "nonpadding_tokens": 9800, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2642655650000165, "nonpadding_tokens": 12547, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2620696120000048, "nonpadding_tokens": 4493, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2598979530000065, "nonpadding_tokens": 4603, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2633967009999765, "nonpadding_tokens": 4776, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2650413470000217, "nonpadding_tokens": 3742, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2659543089999943, "nonpadding_tokens": 1430, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004974599999627571, "prefetch": {"data_wait_seconds": 0.002427057999810245, "consumed_batches": 258, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 808, "tokens": 71167313, "loss_tokens": 62636, "loss": 3.118822691822594, "lr": 0.00029992967781654805, "optimizer_lrs": [0.019995311854436537, 0.00029992967781654805], "gradient_norm": 0.614672064781189, "seconds": 0.2820847539999818, "nonpadding_tokens_per_second": 222046.7398957834, "ranks": [{"compute_seconds": 0.2779594730000099, "nonpadding_tokens": 4784, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2789721899999904, "nonpadding_tokens": 7755, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2782600620000153, "nonpadding_tokens": 11979, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2765958899999532, "nonpadding_tokens": 6311, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.275014667999983, "nonpadding_tokens": 3474, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2777308670000025, "nonpadding_tokens": 5884, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.27890010999999504, "nonpadding_tokens": 7236, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2796985999999606, "nonpadding_tokens": 15213, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004195000000208893, "prefetch": {"data_wait_seconds": 0.0024375539998260365, "consumed_batches": 259, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 809, "tokens": 71219274, "loss_tokens": 51961, "loss": 2.853190373549393, "lr": 0.0002999295349304735, "optimizer_lrs": [0.019995302328698235, 0.0002999295349304735], "gradient_norm": 0.5056222677230835, "seconds": 0.2714898869999729, "nonpadding_tokens_per_second": 191392.02779956657, "ranks": [{"compute_seconds": 0.2661617960000058, "nonpadding_tokens": 4869, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26556635800000095, "nonpadding_tokens": 3289, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26828241300000855, "nonpadding_tokens": 4317, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.26484079099998326, "nonpadding_tokens": 11536, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26921536499997956, "nonpadding_tokens": 3471, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.26644652699997096, "nonpadding_tokens": 1890, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2688297029999944, "nonpadding_tokens": 11713, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2643128999999931, "nonpadding_tokens": 10876, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003536510000117232, "prefetch": {"data_wait_seconds": 0.0024461769998112004, "consumed_batches": 260, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 810, "tokens": 71293692, "loss_tokens": 74418, "loss": 3.2619544330672685, "lr": 0.0002999293300380286, "optimizer_lrs": [0.019995288669201906, 0.0002999293300380286], "gradient_norm": 0.49242493510246277, "seconds": 0.30806322500001215, "nonpadding_tokens_per_second": 241567.2951550678, "ranks": [{"compute_seconds": 0.30552660699999024, "nonpadding_tokens": 14322, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30493354499998304, "nonpadding_tokens": 18447, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3021683199999643, "nonpadding_tokens": 7140, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3020896070000276, "nonpadding_tokens": 3821, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3013331820000076, "nonpadding_tokens": 8422, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3034427499999879, "nonpadding_tokens": 4277, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3011725680000268, "nonpadding_tokens": 5263, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.30572611600001665, "nonpadding_tokens": 12726, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003856750000181819, "prefetch": {"data_wait_seconds": 0.0024551679998268128, "consumed_batches": 261, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 811, "tokens": 71360519, "loss_tokens": 66827, "loss": 3.2106023295224984, "lr": 0.0002999291457921943, "optimizer_lrs": [0.019995276386146288, 0.0002999291457921943], "gradient_norm": 0.8703727126121521, "seconds": 0.28183888000000934, "nonpadding_tokens_per_second": 237110.64988619662, "ranks": [{"compute_seconds": 0.2792760099999896, "nonpadding_tokens": 6530, "peak_allocated_bytes": 15007468032, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.275592147999987, "nonpadding_tokens": 1686, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2751745449999703, "nonpadding_tokens": 5522, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2760645540000155, "nonpadding_tokens": 5635, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2747575860000211, "nonpadding_tokens": 8958, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2769193889999997, "nonpadding_tokens": 2157, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.27437441899996884, "nonpadding_tokens": 17701, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.27949888599999895, "nonpadding_tokens": 18638, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034581799997113194, "prefetch": {"data_wait_seconds": 0.002464092999801437, "consumed_batches": 262, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 812, "tokens": 71435891, "loss_tokens": 75372, "loss": 3.0168483073953194, "lr": 0.0002999289376995543, "optimizer_lrs": [0.01999526251330362, 0.0002999289376995543], "gradient_norm": 1.0658552646636963, "seconds": 0.30540500299997575, "nonpadding_tokens_per_second": 246793.59951417032, "ranks": [{"compute_seconds": 0.3010788099999786, "nonpadding_tokens": 11706, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29912304499998754, "nonpadding_tokens": 4882, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2998938100000146, "nonpadding_tokens": 4140, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.300122760000022, "nonpadding_tokens": 3682, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.302970117999962, "nonpadding_tokens": 5336, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3020103529999574, "nonpadding_tokens": 26767, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.29863689899997325, "nonpadding_tokens": 4497, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3030907480000451, "nonpadding_tokens": 14362, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034718799997790484, "prefetch": {"data_wait_seconds": 0.0024737669997563216, "consumed_batches": 263, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 813, "tokens": 71502792, "loss_tokens": 66901, "loss": 2.911916488542772, "lr": 0.00029992875273873574, "optimizer_lrs": [0.019995250182582386, 0.00029992875273873574], "gradient_norm": 0.5720865726470947, "seconds": 0.29529418200002056, "nonpadding_tokens_per_second": 226557.1219415198, "ranks": [{"compute_seconds": 0.2926648490000048, "nonpadding_tokens": 2000, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2887708189999785, "nonpadding_tokens": 5502, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29006298299998434, "nonpadding_tokens": 14269, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.289431497999999, "nonpadding_tokens": 10277, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28863255900000695, "nonpadding_tokens": 9380, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.29146084500001734, "nonpadding_tokens": 4238, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.29223879700003863, "nonpadding_tokens": 3020, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2891884250000203, "nonpadding_tokens": 18215, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003668279999828883, "prefetch": {"data_wait_seconds": 0.002483993999774725, "consumed_batches": 264, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 814, "tokens": 71557241, "loss_tokens": 54449, "loss": 3.0247226762658634, "lr": 0.00029992860202649493, "optimizer_lrs": [0.019995240135099665, 0.00029992860202649493], "gradient_norm": 0.5847535729408264, "seconds": 0.31086295000000064, "nonpadding_tokens_per_second": 175154.35660634338, "ranks": [{"compute_seconds": 0.30560497999999825, "nonpadding_tokens": 11840, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30361676999996234, "nonpadding_tokens": 13770, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30691667199999983, "nonpadding_tokens": 4080, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3073014809999677, "nonpadding_tokens": 4663, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3085420259999978, "nonpadding_tokens": 8629, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.30636588700002676, "nonpadding_tokens": 2980, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3068861269999843, "nonpadding_tokens": 3519, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.30430888500001174, "nonpadding_tokens": 4968, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036469700000907324, "prefetch": {"data_wait_seconds": 0.0024925409997678116, "consumed_batches": 265, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 815, "tokens": 71629214, "loss_tokens": 71973, "loss": 3.3087268836924957, "lr": 0.00029992840256436523, "optimizer_lrs": [0.01999522683762435, 0.00029992840256436523], "gradient_norm": 0.4992813169956207, "seconds": 0.37683769700004177, "nonpadding_tokens_per_second": 190992.0386759821, "ranks": [{"compute_seconds": 0.37447459700001673, "nonpadding_tokens": 22147, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37028293499997744, "nonpadding_tokens": 5623, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37174326299998484, "nonpadding_tokens": 2788, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.37241100400001415, "nonpadding_tokens": 10065, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3705576199999996, "nonpadding_tokens": 1205, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.37097010200000113, "nonpadding_tokens": 2146, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3730095499999493, "nonpadding_tokens": 7942, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3708082169999898, "nonpadding_tokens": 20057, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003540600000064842, "prefetch": {"data_wait_seconds": 0.0025016739998164894, "consumed_batches": 266, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 816, "tokens": 71688049, "loss_tokens": 58835, "loss": 3.1529805281720065, "lr": 0.00029992823930556156, "optimizer_lrs": [0.019995215953704105, 0.00029992823930556156], "gradient_norm": 0.4614262878894806, "seconds": 0.2763305619999983, "nonpadding_tokens_per_second": 212915.2836883832, "ranks": [{"compute_seconds": 0.2715849659999776, "nonpadding_tokens": 2662, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2736285660000135, "nonpadding_tokens": 1678, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2727949410000292, "nonpadding_tokens": 11676, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27311074899995447, "nonpadding_tokens": 3129, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2736570229999984, "nonpadding_tokens": 13233, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27272516599998653, "nonpadding_tokens": 17403, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.26951000100001465, "nonpadding_tokens": 7036, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2740040389999763, "nonpadding_tokens": 2018, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036213900000348076, "prefetch": {"data_wait_seconds": 0.0025114719998242663, "consumed_batches": 267, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 817, "tokens": 71770355, "loss_tokens": 82306, "loss": 2.9690788034894173, "lr": 0.0002999280106061645, "optimizer_lrs": [0.019995200707077637, 0.0002999280106061645], "gradient_norm": 0.5213837027549744, "seconds": 0.2151530619999562, "nonpadding_tokens_per_second": 382546.2637385879, "ranks": [{"compute_seconds": 0.2126296960000218, "nonpadding_tokens": 5853, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2086314779999725, "nonpadding_tokens": 3248, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21134759199998143, "nonpadding_tokens": 4282, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.21216237599998067, "nonpadding_tokens": 3105, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2084841170000118, "nonpadding_tokens": 20123, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2104699440000104, "nonpadding_tokens": 10309, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.21271804200000588, "nonpadding_tokens": 18092, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.20837412800000266, "nonpadding_tokens": 17294, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003976319999878797, "prefetch": {"data_wait_seconds": 0.002521179999817491, "consumed_batches": 268, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 818, "tokens": 71842173, "loss_tokens": 71818, "loss": 2.83355909207998, "lr": 0.00029992781075204896, "optimizer_lrs": [0.01999518738346993, 0.00029992781075204896], "gradient_norm": 0.5274208188056946, "seconds": 0.2727666189999809, "nonpadding_tokens_per_second": 263294.6812307888, "ranks": [{"compute_seconds": 0.26812635999999657, "nonpadding_tokens": 5523, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26976129999997056, "nonpadding_tokens": 14673, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2673166409999794, "nonpadding_tokens": 12208, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2676269789999992, "nonpadding_tokens": 8025, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27020953900000677, "nonpadding_tokens": 1801, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2676124120000054, "nonpadding_tokens": 2627, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2658662770000433, "nonpadding_tokens": 7700, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.27043137200001865, "nonpadding_tokens": 19261, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038425799999686205, "prefetch": {"data_wait_seconds": 0.0025311849998530533, "consumed_batches": 269, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 819, "tokens": 71954523, "loss_tokens": 112350, "loss": 3.1025180796617713, "lr": 0.0002999274975505366, "optimizer_lrs": [0.01999516650336911, 0.0002999274975505366], "gradient_norm": 0.6227255463600159, "seconds": 0.3110997229999839, "nonpadding_tokens_per_second": 361138.2193355595, "ranks": [{"compute_seconds": 0.3083822789999999, "nonpadding_tokens": 17242, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30868168099999593, "nonpadding_tokens": 10970, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30570381900002985, "nonpadding_tokens": 7594, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.30634104799997885, "nonpadding_tokens": 25390, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30875025199998163, "nonpadding_tokens": 12006, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3058334039999977, "nonpadding_tokens": 9911, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3082410239999831, "nonpadding_tokens": 20233, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3087534350000283, "nonpadding_tokens": 9004, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003973110000288216, "prefetch": {"data_wait_seconds": 0.002539504999859332, "consumed_batches": 270, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 820, "tokens": 72023979, "loss_tokens": 69456, "loss": 2.8599983712710206, "lr": 0.00029992730358690937, "optimizer_lrs": [0.019995153572460626, 0.00029992730358690937], "gradient_norm": 0.7156363725662231, "seconds": 0.4074295269999766, "nonpadding_tokens_per_second": 170473.6534718653, "ranks": [{"compute_seconds": 0.4046529309999869, "nonpadding_tokens": 20143, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.40119276499996204, "nonpadding_tokens": 5097, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4018058390000192, "nonpadding_tokens": 5490, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.40217711400003964, "nonpadding_tokens": 7126, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4009628950000206, "nonpadding_tokens": 5089, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.40213487799996983, "nonpadding_tokens": 8407, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.40476987200003123, "nonpadding_tokens": 13782, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.4051338449999662, "nonpadding_tokens": 4322, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035211799996659465, "prefetch": {"data_wait_seconds": 0.0025487519998392827, "consumed_batches": 271, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 821, "tokens": 72061965, "loss_tokens": 37986, "loss": 2.999170747117359, "lr": 0.00029992719739719306, "optimizer_lrs": [0.019995146493146206, 0.00029992719739719306], "gradient_norm": 0.8676517009735107, "seconds": 0.2286060309999698, "nonpadding_tokens_per_second": 166163.5952203073, "ranks": [{"compute_seconds": 0.22548011800000722, "nonpadding_tokens": 4346, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22186694200001966, "nonpadding_tokens": 2652, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22442500599998993, "nonpadding_tokens": 1948, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.22315417700002627, "nonpadding_tokens": 3704, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.22541096300000163, "nonpadding_tokens": 8840, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2228861310000525, "nonpadding_tokens": 1600, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2261932350000393, "nonpadding_tokens": 10378, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2262157470000261, "nonpadding_tokens": 4518, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004082639999865023, "prefetch": {"data_wait_seconds": 0.002558752999860303, "consumed_batches": 272, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 822, "tokens": 72123259, "loss_tokens": 61294, "loss": 3.2436922863575552, "lr": 0.000299927025886676, "optimizer_lrs": [0.019995135059111737, 0.000299927025886676], "gradient_norm": 0.5185279846191406, "seconds": 0.3675142089999781, "nonpadding_tokens_per_second": 166779.94618707013, "ranks": [{"compute_seconds": 0.36387629900002594, "nonpadding_tokens": 22055, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36331991400004426, "nonpadding_tokens": 9897, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36033018800003447, "nonpadding_tokens": 6126, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.36375900699999875, "nonpadding_tokens": 5352, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35943036399999073, "nonpadding_tokens": 4194, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3611413680000055, "nonpadding_tokens": 5341, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.35926149400000895, "nonpadding_tokens": 5255, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3649785420000171, "nonpadding_tokens": 3074, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000373829000011483, "prefetch": {"data_wait_seconds": 0.002568039999871985, "consumed_batches": 273, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 823, "tokens": 72189779, "loss_tokens": 66520, "loss": 3.3153562838244137, "lr": 0.0002999268395247008, "optimizer_lrs": [0.019995122634980055, 0.0002999268395247008], "gradient_norm": 0.46237194538116455, "seconds": 0.2888926269999956, "nonpadding_tokens_per_second": 230258.5590043494, "ranks": [{"compute_seconds": 0.2864580099999898, "nonpadding_tokens": 8956, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2824145659999999, "nonpadding_tokens": 6372, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2825962519999621, "nonpadding_tokens": 8686, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2850519220000365, "nonpadding_tokens": 18246, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28575452500001575, "nonpadding_tokens": 15191, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.283102663999955, "nonpadding_tokens": 2476, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2814520000000016, "nonpadding_tokens": 3883, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.28261363500001835, "nonpadding_tokens": 2710, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004373720000216963, "prefetch": {"data_wait_seconds": 0.002578633999874569, "consumed_batches": 274, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 824, "tokens": 72258565, "loss_tokens": 68786, "loss": 3.116209602971535, "lr": 0.0002999266465644384, "optimizer_lrs": [0.01999510977096256, 0.0002999266465644384], "gradient_norm": 0.448600709438324, "seconds": 0.274626110999975, "nonpadding_tokens_per_second": 250471.44916240778, "ranks": [{"compute_seconds": 0.2711396669999999, "nonpadding_tokens": 10850, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27211000200003355, "nonpadding_tokens": 3291, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27046628300001885, "nonpadding_tokens": 5747, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2709015659999636, "nonpadding_tokens": 11933, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2717211180000163, "nonpadding_tokens": 11634, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2689052239999796, "nonpadding_tokens": 1961, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.27145242500000677, "nonpadding_tokens": 5171, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.27226598999999396, "nonpadding_tokens": 18199, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037426799997319904, "prefetch": {"data_wait_seconds": 0.00258719499989013, "consumed_batches": 275, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 825, "tokens": 72361181, "loss_tokens": 102616, "loss": 3.386816505223357, "lr": 0.000299926358231172, "optimizer_lrs": [0.0199950905487448, 0.000299926358231172], "gradient_norm": 0.5187059044837952, "seconds": 0.366332854999996, "nonpadding_tokens_per_second": 280116.8352753976, "ranks": [{"compute_seconds": 0.3136950329999877, "nonpadding_tokens": 20106, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3196886730000301, "nonpadding_tokens": 7783, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3168074410000372, "nonpadding_tokens": 28857, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3195902130000263, "nonpadding_tokens": 11534, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3195554469999706, "nonpadding_tokens": 9553, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.31680627700001196, "nonpadding_tokens": 9995, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3194057350000321, "nonpadding_tokens": 2142, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.31992438699995773, "nonpadding_tokens": 12646, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01905236199996807, "prefetch": {"data_wait_seconds": 0.00259633999992559, "consumed_batches": 276, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 826, "tokens": 72421933, "loss_tokens": 60752, "loss": 3.096829631123255, "lr": 0.00029992618726206507, "optimizer_lrs": [0.01999507915080434, 0.00029992618726206507], "gradient_norm": 0.5368216037750244, "seconds": 0.3051574159999859, "nonpadding_tokens_per_second": 199084.1343341392, "ranks": [{"compute_seconds": 0.3009547759999691, "nonpadding_tokens": 12346, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29904863199999454, "nonpadding_tokens": 2547, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29958688599998595, "nonpadding_tokens": 4749, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.30185287300002983, "nonpadding_tokens": 8525, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29848139700004594, "nonpadding_tokens": 15212, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3015124549999655, "nonpadding_tokens": 5834, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3022624800000244, "nonpadding_tokens": 5456, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3028862209999943, "nonpadding_tokens": 6083, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034138500001290595, "prefetch": {"data_wait_seconds": 0.0026047129999398067, "consumed_batches": 277, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 827, "tokens": 72495453, "loss_tokens": 73520, "loss": 3.2438643396354734, "lr": 0.0002999259800960464, "optimizer_lrs": [0.01999506533973643, 0.0002999259800960464], "gradient_norm": 0.5749064087867737, "seconds": 0.2748769940000102, "nonpadding_tokens_per_second": 267465.09022140014, "ranks": [{"compute_seconds": 0.2725529709999819, "nonpadding_tokens": 3193, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26884791100002303, "nonpadding_tokens": 13035, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2694280680000247, "nonpadding_tokens": 11997, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27194056999996974, "nonpadding_tokens": 10189, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2725371319999681, "nonpadding_tokens": 3483, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2715036390000023, "nonpadding_tokens": 4856, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2721588549999865, "nonpadding_tokens": 14122, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.26928899099999626, "nonpadding_tokens": 12645, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003575390000492007, "prefetch": {"data_wait_seconds": 0.0026143679999250935, "consumed_batches": 278, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 828, "tokens": 72538664, "loss_tokens": 43211, "loss": 3.3597738423086714, "lr": 0.0002999258581998683, "optimizer_lrs": [0.019995057213324556, 0.0002999258581998683], "gradient_norm": 0.5120642185211182, "seconds": 0.27739419099998486, "nonpadding_tokens_per_second": 155774.7112303529, "ranks": [{"compute_seconds": 0.2747350459999893, "nonpadding_tokens": 6468, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27044132700001455, "nonpadding_tokens": 3536, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2703777559999594, "nonpadding_tokens": 10045, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27217849400000205, "nonpadding_tokens": 6627, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27425899299998946, "nonpadding_tokens": 4093, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2721836030000304, "nonpadding_tokens": 4586, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2738451879999957, "nonpadding_tokens": 3904, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.27452123000000483, "nonpadding_tokens": 3952, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000348716000019067, "prefetch": {"data_wait_seconds": 0.0026224669999379557, "consumed_batches": 279, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 829, "tokens": 72608123, "loss_tokens": 69459, "loss": 3.176449866108064, "lr": 0.00029992566204922104, "optimizer_lrs": [0.019995044136614737, 0.00029992566204922104], "gradient_norm": 0.4326281249523163, "seconds": 0.31372952300000634, "nonpadding_tokens_per_second": 221397.71652921106, "ranks": [{"compute_seconds": 0.3105143720000001, "nonpadding_tokens": 20662, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31106590500002085, "nonpadding_tokens": 9444, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3101041839999539, "nonpadding_tokens": 14035, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.30848442400002796, "nonpadding_tokens": 5547, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30711134099999526, "nonpadding_tokens": 6601, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.30820133999998234, "nonpadding_tokens": 4816, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3107976679999638, "nonpadding_tokens": 5232, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.31128999200001317, "nonpadding_tokens": 3122, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045849599996472534, "prefetch": {"data_wait_seconds": 0.0026499979999385914, "consumed_batches": 280, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 830, "tokens": 72662056, "loss_tokens": 53933, "loss": 3.1067180575899727, "lr": 0.00029992550956497896, "optimizer_lrs": [0.0199950339709986, 0.00029992550956497896], "gradient_norm": 0.49495047330856323, "seconds": 0.22331597800001646, "nonpadding_tokens_per_second": 241509.81261177838, "ranks": [{"compute_seconds": 0.2190982399999939, "nonpadding_tokens": 4304, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.215718073000005, "nonpadding_tokens": 4968, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2150945029999889, "nonpadding_tokens": 10742, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.21537504699995225, "nonpadding_tokens": 5533, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.22019728199995825, "nonpadding_tokens": 4102, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.21694678900001918, "nonpadding_tokens": 6111, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2198228150000432, "nonpadding_tokens": 7549, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2207381160000068, "nonpadding_tokens": 10624, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004100859999880413, "prefetch": {"data_wait_seconds": 0.0026600479998961646, "consumed_batches": 281, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 831, "tokens": 72738647, "loss_tokens": 76591, "loss": 3.059557013878915, "lr": 0.0002999252927516014, "optimizer_lrs": [0.019995019516773425, 0.0002999252927516014], "gradient_norm": 0.7523433566093445, "seconds": 0.379873947999954, "nonpadding_tokens_per_second": 201622.1444067264, "ranks": [{"compute_seconds": 0.37594829399995433, "nonpadding_tokens": 3959, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3735732430000098, "nonpadding_tokens": 13554, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.372170557000004, "nonpadding_tokens": 2529, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3766837509999732, "nonpadding_tokens": 6206, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3771154730000035, "nonpadding_tokens": 9978, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.37437367100000074, "nonpadding_tokens": 17530, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3768170330000089, "nonpadding_tokens": 13797, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.37180716400001756, "nonpadding_tokens": 9038, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045715300001347714, "prefetch": {"data_wait_seconds": 0.0026697839998774953, "consumed_batches": 282, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 832, "tokens": 72819410, "loss_tokens": 80763, "loss": 3.1393873896462487, "lr": 0.0002999250637869606, "optimizer_lrs": [0.019995004252464043, 0.0002999250637869606], "gradient_norm": 0.559852659702301, "seconds": 0.2944778890000066, "nonpadding_tokens_per_second": 274258.2822576475, "ranks": [{"compute_seconds": 0.29193805399995654, "nonpadding_tokens": 11757, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28929462000002104, "nonpadding_tokens": 13966, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28605278200001294, "nonpadding_tokens": 12262, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2867054709999479, "nonpadding_tokens": 1684, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28506211600000597, "nonpadding_tokens": 3510, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2887669510000137, "nonpadding_tokens": 3076, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.29116770299998507, "nonpadding_tokens": 7805, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2853884670000184, "nonpadding_tokens": 26703, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038548800000626215, "prefetch": {"data_wait_seconds": 0.002678527999833591, "consumed_batches": 283, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 833, "tokens": 72864816, "loss_tokens": 45406, "loss": 2.9180105602783772, "lr": 0.0002999249349062825, "optimizer_lrs": [0.019994995660418836, 0.0002999249349062825], "gradient_norm": 0.5272159576416016, "seconds": 0.27087545000000546, "nonpadding_tokens_per_second": 167626.8558114037, "ranks": [{"compute_seconds": 0.2664519630000086, "nonpadding_tokens": 10100, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26492693900001996, "nonpadding_tokens": 6283, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26755221199999824, "nonpadding_tokens": 4914, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2638940179999736, "nonpadding_tokens": 2919, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2684624880000115, "nonpadding_tokens": 3054, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.26501308599995355, "nonpadding_tokens": 9737, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2684116529999869, "nonpadding_tokens": 1329, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2624106220000044, "nonpadding_tokens": 7070, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037740400000529917, "prefetch": {"data_wait_seconds": 0.0026878179998135465, "consumed_batches": 284, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 834, "tokens": 72936237, "loss_tokens": 71421, "loss": 2.8090007228266196, "lr": 0.00029992473196045093, "optimizer_lrs": [0.01999498213069673, 0.00029992473196045093], "gradient_norm": 0.494127094745636, "seconds": 0.28807130199999165, "nonpadding_tokens_per_second": 247928.20216434496, "ranks": [{"compute_seconds": 0.2818031510000196, "nonpadding_tokens": 11774, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28156283499998835, "nonpadding_tokens": 6268, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2843211169999904, "nonpadding_tokens": 9566, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.28481331199998294, "nonpadding_tokens": 10269, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.285524310000028, "nonpadding_tokens": 5525, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2822419859999741, "nonpadding_tokens": 13065, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2850407939999968, "nonpadding_tokens": 1531, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.28573004399999036, "nonpadding_tokens": 13423, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003453530000001592, "prefetch": {"data_wait_seconds": 0.0026969029998014094, "consumed_batches": 285, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 835, "tokens": 72999803, "loss_tokens": 63566, "loss": 3.167587625460152, "lr": 0.0002999245511045719, "optimizer_lrs": [0.019994970073638128, 0.0002999245511045719], "gradient_norm": 0.530724287033081, "seconds": 0.2900684040000101, "nonpadding_tokens_per_second": 219141.41327849615, "ranks": [{"compute_seconds": 0.2873051999999916, "nonpadding_tokens": 3983, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2872925940000073, "nonpadding_tokens": 8016, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28420280000000275, "nonpadding_tokens": 13956, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2866064680000022, "nonpadding_tokens": 2449, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2877292389999866, "nonpadding_tokens": 6476, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2849811659999659, "nonpadding_tokens": 4430, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.28723862099997177, "nonpadding_tokens": 12192, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.28389676699998745, "nonpadding_tokens": 12064, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003827180000257613, "prefetch": {"data_wait_seconds": 0.002705142999801069, "consumed_batches": 286, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 836, "tokens": 73089917, "loss_tokens": 90114, "loss": 2.7747544083050357, "lr": 0.00029992429434340005, "optimizer_lrs": [0.01999495295622667, 0.00029992429434340005], "gradient_norm": 0.5586879849433899, "seconds": 0.38511052799998424, "nonpadding_tokens_per_second": 233995.16099441375, "ranks": [{"compute_seconds": 0.3824053079999885, "nonpadding_tokens": 22648, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3783555100000058, "nonpadding_tokens": 20359, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3812326759999678, "nonpadding_tokens": 10255, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.3817875240000035, "nonpadding_tokens": 15110, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3824610679999978, "nonpadding_tokens": 9660, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3819032100000186, "nonpadding_tokens": 4160, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.3793719330000158, "nonpadding_tokens": 2936, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3802801470000077, "nonpadding_tokens": 4986, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039303800002699063, "prefetch": {"data_wait_seconds": 0.002714528999845811, "consumed_batches": 287, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 837, "tokens": 73135370, "loss_tokens": 45453, "loss": 3.143563818669835, "lr": 0.0002999241646690628, "optimizer_lrs": [0.019994944311270853, 0.0002999241646690628], "gradient_norm": 0.47734490036964417, "seconds": 0.24036699499998804, "nonpadding_tokens_per_second": 189098.34105968775, "ranks": [{"compute_seconds": 0.23768664900001113, "nonpadding_tokens": 1435, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23802135500000077, "nonpadding_tokens": 9996, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2369652149999979, "nonpadding_tokens": 3190, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23736376499999778, "nonpadding_tokens": 3584, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23792880000002015, "nonpadding_tokens": 15894, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.23692104500003097, "nonpadding_tokens": 1891, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.23400623300000234, "nonpadding_tokens": 7358, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.23460414599998103, "nonpadding_tokens": 2105, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003542430000038621, "prefetch": {"data_wait_seconds": 0.002723620999873333, "consumed_batches": 288, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 838, "tokens": 73183439, "loss_tokens": 48069, "loss": 2.8969947497347563, "lr": 0.00029992402741076205, "optimizer_lrs": [0.019994935160717472, 0.00029992402741076205], "gradient_norm": 0.5652803182601929, "seconds": 0.2503806779999991, "nonpadding_tokens_per_second": 191983.66417076392, "ranks": [{"compute_seconds": 0.2477773990000287, "nonpadding_tokens": 8737, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2479083320000086, "nonpadding_tokens": 13170, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24500301499995203, "nonpadding_tokens": 5316, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2457768680000072, "nonpadding_tokens": 6098, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2442746189999525, "nonpadding_tokens": 3425, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.24686705699997447, "nonpadding_tokens": 5953, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.24795926199999485, "nonpadding_tokens": 2277, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.24412095499997122, "nonpadding_tokens": 3093, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003749210000023595, "prefetch": {"data_wait_seconds": 0.0027323859998915623, "consumed_batches": 289, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 839, "tokens": 73261841, "loss_tokens": 78402, "loss": 3.0467550253820055, "lr": 0.0002999238032720954, "optimizer_lrs": [0.019994920218139693, 0.0002999238032720954], "gradient_norm": 0.44873151183128357, "seconds": 0.27578653000000486, "nonpadding_tokens_per_second": 284285.09543232084, "ranks": [{"compute_seconds": 0.23604123599994864, "nonpadding_tokens": 19864, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2422217440000054, "nonpadding_tokens": 4776, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23880911899999546, "nonpadding_tokens": 4517, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.24528991799996902, "nonpadding_tokens": 3756, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23787270600001875, "nonpadding_tokens": 5420, "peak_allocated_bytes": 14978018816, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2428784659999792, "nonpadding_tokens": 15657, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.24519777199998316, "nonpadding_tokens": 12499, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.24608297300000004, "nonpadding_tokens": 11913, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.016409000999999535, "prefetch": {"data_wait_seconds": 0.0027416459998477194, "consumed_batches": 290, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 840, "tokens": 73331559, "loss_tokens": 69718, "loss": 3.2291458984767205, "lr": 0.00029992360368234186, "optimizer_lrs": [0.019994906912156125, 0.00029992360368234186], "gradient_norm": 0.4342266023159027, "seconds": 0.2996442049999928, "nonpadding_tokens_per_second": 232669.27521592376, "ranks": [{"compute_seconds": 0.2942427779999548, "nonpadding_tokens": 3517, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2929626760000019, "nonpadding_tokens": 5535, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.295441387999972, "nonpadding_tokens": 9560, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.28989834599997266, "nonpadding_tokens": 7202, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2968274009999732, "nonpadding_tokens": 11411, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.29404904099999385, "nonpadding_tokens": 17827, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2883566289999635, "nonpadding_tokens": 12530, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.29706838000004154, "nonpadding_tokens": 2136, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004065969999942354, "prefetch": {"data_wait_seconds": 0.00275021699981437, "consumed_batches": 291, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 841, "tokens": 73389279, "loss_tokens": 57720, "loss": 2.833877717862093, "lr": 0.0002999234382431652, "optimizer_lrs": [0.01999489588287768, 0.0002999234382431652], "gradient_norm": 1.4024279117584229, "seconds": 0.2916274599999724, "nonpadding_tokens_per_second": 197923.7483329089, "ranks": [{"compute_seconds": 0.2893864789999725, "nonpadding_tokens": 2040, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28632038599999987, "nonpadding_tokens": 10635, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2887192270000014, "nonpadding_tokens": 2543, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.28719205200002307, "nonpadding_tokens": 6142, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.285422307000033, "nonpadding_tokens": 18319, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.28859877700000425, "nonpadding_tokens": 7720, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2853923160000136, "nonpadding_tokens": 6686, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2861000190000027, "nonpadding_tokens": 3635, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033863999999539374, "prefetch": {"data_wait_seconds": 0.0027579149998047114, "consumed_batches": 292, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 842, "tokens": 73445924, "loss_tokens": 56645, "loss": 3.008382524936005, "lr": 0.00029992327571127137, "optimizer_lrs": [0.019994885047418094, 0.00029992327571127137], "gradient_norm": 0.5117248892784119, "seconds": 0.2524012599999992, "nonpadding_tokens_per_second": 224424.39471181793, "ranks": [{"compute_seconds": 0.24999891900000648, "nonpadding_tokens": 2604, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24630673400002934, "nonpadding_tokens": 8524, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24594365299998344, "nonpadding_tokens": 5694, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.24952664599999252, "nonpadding_tokens": 13693, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24389085000001387, "nonpadding_tokens": 7099, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.24783250400002999, "nonpadding_tokens": 6059, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2435154940000075, "nonpadding_tokens": 7259, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2441048539999997, "nonpadding_tokens": 5713, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043344900001329734, "prefetch": {"data_wait_seconds": 0.0027672729998045043, "consumed_batches": 293, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 843, "tokens": 73490632, "loss_tokens": 44708, "loss": 2.9240023457770423, "lr": 0.00029992314730864283, "optimizer_lrs": [0.019994876487242855, 0.00029992314730864283], "gradient_norm": 0.4996514320373535, "seconds": 0.2562198470000112, "nonpadding_tokens_per_second": 174490.77627463435, "ranks": [{"compute_seconds": 0.2534540870000228, "nonpadding_tokens": 2622, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2539330829999926, "nonpadding_tokens": 2527, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25303360899999916, "nonpadding_tokens": 4628, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2513439299999618, "nonpadding_tokens": 8042, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2536978389999831, "nonpadding_tokens": 4908, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2522406069999761, "nonpadding_tokens": 6316, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.25349241200001416, "nonpadding_tokens": 9312, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2537916509999718, "nonpadding_tokens": 6353, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003666210000119463, "prefetch": {"data_wait_seconds": 0.0027764069997857632, "consumed_batches": 294, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 844, "tokens": 73550400, "loss_tokens": 59768, "loss": 3.2000584551934144, "lr": 0.00029992297548563036, "optimizer_lrs": [0.01999486503237536, 0.00029992297548563036], "gradient_norm": 0.5668644905090332, "seconds": 0.3266144259999919, "nonpadding_tokens_per_second": 182992.52954614282, "ranks": [{"compute_seconds": 0.3243010989999675, "nonpadding_tokens": 6264, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3209651359999839, "nonpadding_tokens": 1655, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3230811540000218, "nonpadding_tokens": 14185, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.32186894399995936, "nonpadding_tokens": 9442, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3205373509999845, "nonpadding_tokens": 2752, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3216399980000233, "nonpadding_tokens": 10050, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.32046067599998196, "nonpadding_tokens": 13527, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3211006689999749, "nonpadding_tokens": 1893, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040637699999024335, "prefetch": {"data_wait_seconds": 0.0027850019998396647, "consumed_batches": 295, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 845, "tokens": 73610861, "loss_tokens": 60461, "loss": 3.0263186186136517, "lr": 0.0002999228014752029, "optimizer_lrs": [0.01999485343168019, 0.0002999228014752029], "gradient_norm": 0.493669718503952, "seconds": 0.25868791999999985, "nonpadding_tokens_per_second": 233721.77564379518, "ranks": [{"compute_seconds": 0.22987082499997769, "nonpadding_tokens": 11076, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23363663300000326, "nonpadding_tokens": 3500, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23097034300002406, "nonpadding_tokens": 12195, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23376146200001813, "nonpadding_tokens": 11515, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23402091100001599, "nonpadding_tokens": 5077, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.23126588599996012, "nonpadding_tokens": 3904, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19056820224}, {"compute_seconds": 0.2298097279999638, "nonpadding_tokens": 6828, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.23453591499998083, "nonpadding_tokens": 6366, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.014352297000016279, "prefetch": {"data_wait_seconds": 0.0027937929998103073, "consumed_batches": 296, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 846, "tokens": 73685090, "loss_tokens": 74229, "loss": 3.074300896549866, "lr": 0.0002999225875712217, "optimizer_lrs": [0.019994839171414782, 0.0002999225875712217], "gradient_norm": 0.4718044698238373, "seconds": 0.30538941699995803, "nonpadding_tokens_per_second": 243063.43267949656, "ranks": [{"compute_seconds": 0.3010002389999613, "nonpadding_tokens": 8701, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29904673700002604, "nonpadding_tokens": 14518, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2971270399999639, "nonpadding_tokens": 13643, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2974588870000048, "nonpadding_tokens": 7705, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2962640169999986, "nonpadding_tokens": 2734, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2996308269999872, "nonpadding_tokens": 11056, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3025278549999939, "nonpadding_tokens": 4161, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.30304046499998094, "nonpadding_tokens": 11711, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003756300000077317, "prefetch": {"data_wait_seconds": 0.0028025509997888776, "consumed_batches": 297, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 847, "tokens": 73753128, "loss_tokens": 68038, "loss": 3.187460500014698, "lr": 0.0002999223912478339, "optimizer_lrs": [0.019994826083188925, 0.0002999223912478339], "gradient_norm": 0.5393101572990417, "seconds": 0.2948348389999751, "nonpadding_tokens_per_second": 230766.48686014256, "ranks": [{"compute_seconds": 0.29194514099998514, "nonpadding_tokens": 6907, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28856203899999855, "nonpadding_tokens": 5260, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28906911799998625, "nonpadding_tokens": 14502, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.29157183099999884, "nonpadding_tokens": 4991, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28802909199998794, "nonpadding_tokens": 10243, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.28957945500002324, "nonpadding_tokens": 9257, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.28804995699999836, "nonpadding_tokens": 2182, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2924822429999949, "nonpadding_tokens": 14696, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038224999997282794, "prefetch": {"data_wait_seconds": 0.0028125119997781667, "consumed_batches": 298, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 848, "tokens": 73858277, "loss_tokens": 105149, "loss": 3.8127389466376287, "lr": 0.00029992208735190326, "optimizer_lrs": [0.019994805823460218, 0.00029992208735190326], "gradient_norm": 0.7493481040000916, "seconds": 0.3538519559999713, "nonpadding_tokens_per_second": 297155.3448188613, "ranks": [{"compute_seconds": 0.32157877100001997, "nonpadding_tokens": 22372, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3346904619999691, "nonpadding_tokens": 10596, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33453346900000724, "nonpadding_tokens": 11431, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.33444178700000293, "nonpadding_tokens": 8921, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33069074600001613, "nonpadding_tokens": 11677, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.33233328500000425, "nonpadding_tokens": 15573, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3348094610000203, "nonpadding_tokens": 5210, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3320232689999898, "nonpadding_tokens": 19369, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005334900000093512, "prefetch": {"data_wait_seconds": 0.0028222169998031177, "consumed_batches": 299, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 849, "tokens": 73901139, "loss_tokens": 42862, "loss": 2.968592882389996, "lr": 0.00029992196330416, "optimizer_lrs": [0.01999479755361067, 0.00029992196330416], "gradient_norm": 0.5692755579948425, "seconds": 0.20373843900000566, "nonpadding_tokens_per_second": 210377.58122805099, "ranks": [{"compute_seconds": 0.201456324999981, "nonpadding_tokens": 8840, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20124534900003255, "nonpadding_tokens": 2451, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1999643350000042, "nonpadding_tokens": 3009, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.19857954700000846, "nonpadding_tokens": 5380, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.19644506599996703, "nonpadding_tokens": 2057, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.19953328000002557, "nonpadding_tokens": 8319, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.20042751299996553, "nonpadding_tokens": 10421, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.1973108459999935, "nonpadding_tokens": 2385, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034940799997684735, "prefetch": {"data_wait_seconds": 0.0028316219998174574, "consumed_batches": 300, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 850, "tokens": 73953000, "loss_tokens": 51861, "loss": 3.190016944331964, "lr": 0.00029992181308035346, "optimizer_lrs": [0.01999478753869023, 0.00029992181308035346], "gradient_norm": 0.7904204726219177, "seconds": 0.30911439200002633, "nonpadding_tokens_per_second": 167772.84184165575, "ranks": [{"compute_seconds": 0.3058052790000261, "nonpadding_tokens": 9500, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3057176749999826, "nonpadding_tokens": 2854, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3033290999999849, "nonpadding_tokens": 6359, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.30375232500000493, "nonpadding_tokens": 6517, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30645425199998044, "nonpadding_tokens": 11103, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.303539915999977, "nonpadding_tokens": 2691, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.30611025299998573, "nonpadding_tokens": 1864, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3067799649999756, "nonpadding_tokens": 10973, "peak_allocated_bytes": 14996449792, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041435200000705663, "prefetch": {"data_wait_seconds": 0.0028420609998107693, "consumed_batches": 301, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 851, "tokens": 74027760, "loss_tokens": 74760, "loss": 2.9264903942616374, "lr": 0.0002999215962717384, "optimizer_lrs": [0.01999477308478256, 0.0002999215962717384], "gradient_norm": 0.50612872838974, "seconds": 0.31543028300001197, "nonpadding_tokens_per_second": 237009.5835091305, "ranks": [{"compute_seconds": 0.31311399300000176, "nonpadding_tokens": 9738, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3120495019999794, "nonpadding_tokens": 13525, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3096570850000262, "nonpadding_tokens": 1618, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.31235170299999027, "nonpadding_tokens": 7331, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3126987750000012, "nonpadding_tokens": 8797, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3097050000000081, "nonpadding_tokens": 5554, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.31231067999999595, "nonpadding_tokens": 13358, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.31295351400001437, "nonpadding_tokens": 14839, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038218000003098496, "prefetch": {"data_wait_seconds": 0.0028507249997602457, "consumed_batches": 302, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 852, "tokens": 74083992, "loss_tokens": 56232, "loss": 2.894543476134585, "lr": 0.000299921432997715, "optimizer_lrs": [0.01999476219984767, 0.000299921432997715], "gradient_norm": 0.5298104286193848, "seconds": 0.30727973000000475, "nonpadding_tokens_per_second": 182999.379750819, "ranks": [{"compute_seconds": 0.30396763499999224, "nonpadding_tokens": 6860, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3013139490000185, "nonpadding_tokens": 12063, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3038318810000078, "nonpadding_tokens": 4398, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.30187868699999854, "nonpadding_tokens": 15290, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30106970999997884, "nonpadding_tokens": 1769, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3032516810000061, "nonpadding_tokens": 5948, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3041894099999922, "nonpadding_tokens": 3490, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.30503458100002945, "nonpadding_tokens": 6414, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003666279999947619, "prefetch": {"data_wait_seconds": 0.0028597199997904, "consumed_batches": 303, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 853, "tokens": 74127517, "loss_tokens": 43525, "loss": 3.1917658673176335, "lr": 0.00029992130650289644, "optimizer_lrs": [0.019994753766859764, 0.00029992130650289644], "gradient_norm": 0.5282361507415771, "seconds": 0.273150950999991, "nonpadding_tokens_per_second": 159344.1276358633, "ranks": [{"compute_seconds": 0.27059558199999856, "nonpadding_tokens": 3962, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2703065339999853, "nonpadding_tokens": 2397, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2676946430000271, "nonpadding_tokens": 1673, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27006736699996736, "nonpadding_tokens": 5385, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2666493329999753, "nonpadding_tokens": 9576, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.26945546499996453, "nonpadding_tokens": 10565, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2663196249999942, "nonpadding_tokens": 3339, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.27087153099995476, "nonpadding_tokens": 6628, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003693670000188831, "prefetch": {"data_wait_seconds": 0.002868957999751842, "consumed_batches": 304, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 854, "tokens": 74189450, "loss_tokens": 61933, "loss": 2.8948924644373757, "lr": 0.0002999211263343614, "optimizer_lrs": [0.019994741755624094, 0.0002999211263343614], "gradient_norm": 0.6087215542793274, "seconds": 0.29092612699997744, "nonpadding_tokens_per_second": 212882.2207845389, "ranks": [{"compute_seconds": 0.2886658800000532, "nonpadding_tokens": 9780, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28812020399999483, "nonpadding_tokens": 4225, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2855634179999811, "nonpadding_tokens": 5146, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.288163609000037, "nonpadding_tokens": 7553, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2845543189999944, "nonpadding_tokens": 4051, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.28722950999997465, "nonpadding_tokens": 10848, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2882553869999924, "nonpadding_tokens": 7449, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.28503305599997475, "nonpadding_tokens": 12881, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003931709999847044, "prefetch": {"data_wait_seconds": 0.00287766199973305, "consumed_batches": 305, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 855, "tokens": 74263659, "loss_tokens": 74209, "loss": 2.814709132315487, "lr": 0.0002999209101826332, "optimizer_lrs": [0.019994727345508882, 0.0002999209101826332], "gradient_norm": 0.6507410407066345, "seconds": 0.22793638600001032, "nonpadding_tokens_per_second": 325568.9067562765, "ranks": [{"compute_seconds": 0.2112428780000073, "nonpadding_tokens": 2337, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20902658799997198, "nonpadding_tokens": 3115, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20927264999994577, "nonpadding_tokens": 17956, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.21166200600004004, "nonpadding_tokens": 6212, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21251771499998995, "nonpadding_tokens": 10205, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2107523700000229, "nonpadding_tokens": 16313, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.20829628900003172, "nonpadding_tokens": 12156, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.20906193100000792, "nonpadding_tokens": 5915, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.010343981000005442, "prefetch": {"data_wait_seconds": 0.002885442999755128, "consumed_batches": 306, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 856, "tokens": 74337892, "loss_tokens": 74233, "loss": 2.8294045438012745, "lr": 0.0002999206936651679, "optimizer_lrs": [0.019994712911011195, 0.0002999206936651679], "gradient_norm": 0.6439790725708008, "seconds": 0.2861332610000318, "nonpadding_tokens_per_second": 259435.0609242585, "ranks": [{"compute_seconds": 0.2838289139999688, "nonpadding_tokens": 4756, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2798618319999946, "nonpadding_tokens": 9669, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2825400969999805, "nonpadding_tokens": 17561, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2829124870000328, "nonpadding_tokens": 9012, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2795964840000238, "nonpadding_tokens": 7796, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.28195574500000475, "nonpadding_tokens": 6394, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.28341925299997683, "nonpadding_tokens": 9034, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2797922750000339, "nonpadding_tokens": 10011, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035545000002912275, "prefetch": {"data_wait_seconds": 0.002893307999798367, "consumed_batches": 307, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 857, "tokens": 74396538, "loss_tokens": 58646, "loss": 3.1141576684684376, "lr": 0.0002999205224015301, "optimizer_lrs": [0.01999470149343534, 0.0002999205224015301], "gradient_norm": 0.630137026309967, "seconds": 0.2753891899999985, "nonpadding_tokens_per_second": 212956.79761431564, "ranks": [{"compute_seconds": 0.27308490600000823, "nonpadding_tokens": 4370, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26951744600000893, "nonpadding_tokens": 11880, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2724063219999948, "nonpadding_tokens": 4487, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27075082899995095, "nonpadding_tokens": 3282, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2694466039999952, "nonpadding_tokens": 6779, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2717128680000087, "nonpadding_tokens": 7736, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.26926223100002744, "nonpadding_tokens": 10092, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2698958779999998, "nonpadding_tokens": 10020, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038746399997080516, "prefetch": {"data_wait_seconds": 0.0029027229998206394, "consumed_batches": 308, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 858, "tokens": 74466773, "loss_tokens": 70235, "loss": 2.941485504022211, "lr": 0.0002999203170515759, "optimizer_lrs": [0.019994687803438396, 0.0002999203170515759], "gradient_norm": 0.5269082188606262, "seconds": 0.28936247199999343, "nonpadding_tokens_per_second": 242723.25127220227, "ranks": [{"compute_seconds": 0.28696461100003035, "nonpadding_tokens": 9698, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2832149529999697, "nonpadding_tokens": 6128, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28358275599998706, "nonpadding_tokens": 10021, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.28393003400003636, "nonpadding_tokens": 5097, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28680739499998253, "nonpadding_tokens": 18413, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.28398067500000934, "nonpadding_tokens": 7470, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.282206551999991, "nonpadding_tokens": 6020, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2870456899999567, "nonpadding_tokens": 7388, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035971599999129467, "prefetch": {"data_wait_seconds": 0.0029105379998242142, "consumed_batches": 309, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 859, "tokens": 74497843, "loss_tokens": 31070, "loss": 3.12686800973608, "lr": 0.0002999202261259977, "optimizer_lrs": [0.019994681741733182, 0.0002999202261259977], "gradient_norm": 0.6055168509483337, "seconds": 0.24831268799999862, "nonpadding_tokens_per_second": 125124.49625610824, "ranks": [{"compute_seconds": 0.24576157899997497, "nonpadding_tokens": 1201, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24260466999999153, "nonpadding_tokens": 2393, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2451353689999678, "nonpadding_tokens": 2130, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.24297046800001, "nonpadding_tokens": 10284, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24196556999999075, "nonpadding_tokens": 5135, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.243153379999967, "nonpadding_tokens": 2114, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2411073239999837, "nonpadding_tokens": 4743, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2428339179999739, "nonpadding_tokens": 3070, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003812209999978222, "prefetch": {"data_wait_seconds": 0.002919716999841171, "consumed_batches": 310, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 860, "tokens": 74560741, "loss_tokens": 62898, "loss": 3.1192076357753824, "lr": 0.00029992004189789955, "optimizer_lrs": [0.019994669459859973, 0.00029992004189789955], "gradient_norm": 0.4610544741153717, "seconds": 0.18484661700000515, "nonpadding_tokens_per_second": 340271.3072103356, "ranks": [{"compute_seconds": 0.17893237200001977, "nonpadding_tokens": 1717, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.1786442549999947, "nonpadding_tokens": 5170, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.17778806199999053, "nonpadding_tokens": 9383, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.17812613699999247, "nonpadding_tokens": 24122, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.1822886799999992, "nonpadding_tokens": 3126, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.1792719460000285, "nonpadding_tokens": 2824, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.1769294000000059, "nonpadding_tokens": 8149, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.1825498009999933, "nonpadding_tokens": 8407, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039645999999038395, "prefetch": {"data_wait_seconds": 0.0029415929997895773, "consumed_batches": 311, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 861, "tokens": 74614257, "loss_tokens": 53516, "loss": 2.9698971452462817, "lr": 0.000299919884982404, "optimizer_lrs": [0.019994658998826933, 0.000299919884982404], "gradient_norm": 0.7002259492874146, "seconds": 0.19620267999999896, "nonpadding_tokens_per_second": 272758.76150111854, "ranks": [{"compute_seconds": 0.1938575300000025, "nonpadding_tokens": 3873, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.1928528710000137, "nonpadding_tokens": 10611, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19233531099996526, "nonpadding_tokens": 3561, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.190123893999953, "nonpadding_tokens": 14695, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.18867015400002174, "nonpadding_tokens": 3146, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.19072039600001744, "nonpadding_tokens": 4063, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.19296907599999713, "nonpadding_tokens": 7840, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.19355706800001826, "nonpadding_tokens": 5727, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034444499999608524, "prefetch": {"data_wait_seconds": 0.002950244999809115, "consumed_batches": 312, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 862, "tokens": 74673809, "loss_tokens": 59552, "loss": 2.8891415275725416, "lr": 0.0002999197101878452, "optimizer_lrs": [0.019994647345856348, 0.0002999197101878452], "gradient_norm": 0.5975698232650757, "seconds": 0.27687821299997495, "nonpadding_tokens_per_second": 215083.73430597584, "ranks": [{"compute_seconds": 0.274347726999963, "nonpadding_tokens": 3051, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27018089500000997, "nonpadding_tokens": 8489, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2728733610000518, "nonpadding_tokens": 6022, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27333142399999133, "nonpadding_tokens": 8507, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27431135399996265, "nonpadding_tokens": 6911, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.27098525100001325, "nonpadding_tokens": 17441, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2694150779999518, "nonpadding_tokens": 3445, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2697368890000007, "nonpadding_tokens": 5686, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042553999998062864, "prefetch": {"data_wait_seconds": 0.0029601799997891476, "consumed_batches": 313, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 863, "tokens": 74740375, "loss_tokens": 66566, "loss": 2.566376321996214, "lr": 0.0002999195145807045, "optimizer_lrs": [0.0199946343053803, 0.0002999195145807045], "gradient_norm": 0.5491644144058228, "seconds": 0.2783465389999833, "nonpadding_tokens_per_second": 239147.93494164478, "ranks": [{"compute_seconds": 0.2757069719999663, "nonpadding_tokens": 2789, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2719858109999791, "nonpadding_tokens": 4679, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27245959200001835, "nonpadding_tokens": 14185, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.27542143099998384, "nonpadding_tokens": 12908, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2758429529999944, "nonpadding_tokens": 9813, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.274524600999996, "nonpadding_tokens": 2194, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.27121050099998456, "nonpadding_tokens": 9037, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2720702439999627, "nonpadding_tokens": 10961, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004014389999724699, "prefetch": {"data_wait_seconds": 0.002970053999789002, "consumed_batches": 314, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 864, "tokens": 74768586, "loss_tokens": 28211, "loss": 2.894102145439722, "lr": 0.0002999194316096502, "optimizer_lrs": [0.019994628773976682, 0.0002999194316096502], "gradient_norm": 0.6807490587234497, "seconds": 0.24118293800000856, "nonpadding_tokens_per_second": 116969.30236416226, "ranks": [{"compute_seconds": 0.23727499900002158, "nonpadding_tokens": 2168, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2382584970000039, "nonpadding_tokens": 3572, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23513810299999705, "nonpadding_tokens": 2898, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.23883809700004122, "nonpadding_tokens": 4178, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23848711499999808, "nonpadding_tokens": 3489, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.23582241799999792, "nonpadding_tokens": 6953, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.23381617600000482, "nonpadding_tokens": 3375, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.23861690100000033, "nonpadding_tokens": 1578, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041295299996590984, "prefetch": {"data_wait_seconds": 0.0029797579998245283, "consumed_batches": 315, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 865, "tokens": 74798185, "loss_tokens": 29599, "loss": 3.0112031339741208, "lr": 0.00029991934451042746, "optimizer_lrs": [0.019994622967361833, 0.00029991934451042746], "gradient_norm": 0.6073363423347473, "seconds": 0.29890182999997705, "nonpadding_tokens_per_second": 99025.82396368156, "ranks": [{"compute_seconds": 0.28810025199999245, "nonpadding_tokens": 8282, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2881077260000211, "nonpadding_tokens": 7782, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2911162489999697, "nonpadding_tokens": 1923, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2958510659999547, "nonpadding_tokens": 1574, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.286438783000051, "nonpadding_tokens": 3391, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.28911647799998264, "nonpadding_tokens": 2158, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.28574177000001555, "nonpadding_tokens": 1692, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2924372149999499, "nonpadding_tokens": 2797, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039684900002612267, "prefetch": {"data_wait_seconds": 0.0029898559998429164, "consumed_batches": 316, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 866, "tokens": 74864921, "loss_tokens": 66736, "loss": 2.870569529564253, "lr": 0.0002999191479577606, "optimizer_lrs": [0.01999460986385071, 0.0002999191479577606], "gradient_norm": 0.6040683388710022, "seconds": 0.2813018449999731, "nonpadding_tokens_per_second": 237239.82329375186, "ranks": [{"compute_seconds": 0.27525242299998354, "nonpadding_tokens": 3595, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27520902099996647, "nonpadding_tokens": 16683, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2732071120000228, "nonpadding_tokens": 4794, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2773687790000281, "nonpadding_tokens": 6238, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2718175040000119, "nonpadding_tokens": 16499, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2752503800000454, "nonpadding_tokens": 8800, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2774638550000077, "nonpadding_tokens": 3415, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2788143850000324, "nonpadding_tokens": 6712, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003985529999681603, "prefetch": {"data_wait_seconds": 0.0029991099998056825, "consumed_batches": 317, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 867, "tokens": 74920454, "loss_tokens": 55533, "loss": 2.9206071727621414, "lr": 0.0002999189842181861, "optimizer_lrs": [0.019994598947879076, 0.0002999189842181861], "gradient_norm": 0.5629538297653198, "seconds": 0.26794542900000806, "nonpadding_tokens_per_second": 207254.88845715046, "ranks": [{"compute_seconds": 0.26335590000002185, "nonpadding_tokens": 3061, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2641382150000027, "nonpadding_tokens": 13425, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2606045019999783, "nonpadding_tokens": 8292, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.26142596599999024, "nonpadding_tokens": 2308, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25966508499999463, "nonpadding_tokens": 9261, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2612599749999731, "nonpadding_tokens": 4361, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.25982526199999256, "nonpadding_tokens": 3068, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.26546148900001754, "nonpadding_tokens": 11757, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000407727000037994, "prefetch": {"data_wait_seconds": 0.003006313999847521, "consumed_batches": 318, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 868, "tokens": 75001615, "loss_tokens": 81161, "loss": 2.893978673562425, "lr": 0.0002999187446163731, "optimizer_lrs": [0.019994582974424874, 0.0002999187446163731], "gradient_norm": 0.47191715240478516, "seconds": 0.3017761809999797, "nonpadding_tokens_per_second": 268944.3538289241, "ranks": [{"compute_seconds": 0.2974449369999661, "nonpadding_tokens": 3300, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2964710109999942, "nonpadding_tokens": 12807, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2980131279999796, "nonpadding_tokens": 10859, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.2982708249999746, "nonpadding_tokens": 11686, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2924405600000455, "nonpadding_tokens": 16405, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.29543891299999814, "nonpadding_tokens": 9764, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.29223412599998255, "nonpadding_tokens": 5242, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.29933922899999743, "nonpadding_tokens": 11098, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003682359999856999, "prefetch": {"data_wait_seconds": 0.0030143429998616966, "consumed_batches": 319, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 869, "tokens": 75062430, "loss_tokens": 60815, "loss": 3.311095895338321, "lr": 0.00029991856484779965, "optimizer_lrs": [0.01999457098985331, 0.00029991856484779965], "gradient_norm": 0.48543691635131836, "seconds": 0.29132821499996453, "nonpadding_tokens_per_second": 208750.80705796863, "ranks": [{"compute_seconds": 0.2847211690000222, "nonpadding_tokens": 7804, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2858326119999788, "nonpadding_tokens": 4136, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2882143429999928, "nonpadding_tokens": 3860, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18358468608}, {"compute_seconds": 0.28402360199999066, "nonpadding_tokens": 1548, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28215258900002027, "nonpadding_tokens": 4366, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2866190010000196, "nonpadding_tokens": 5199, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2887837820000527, "nonpadding_tokens": 13282, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.28317310599999246, "nonpadding_tokens": 20620, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044959300004165925, "prefetch": {"data_wait_seconds": 0.003025504999826012, "consumed_batches": 320, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 870, "tokens": 75109475, "loss_tokens": 47045, "loss": 3.056163314379849, "lr": 0.000299918425646993, "optimizer_lrs": [0.019994561709799536, 0.000299918425646993], "gradient_norm": 0.5865728855133057, "seconds": 0.2860112489999551, "nonpadding_tokens_per_second": 164486.53738093842, "ranks": [{"compute_seconds": 0.28065647699997953, "nonpadding_tokens": 11510, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2798624410000343, "nonpadding_tokens": 8126, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27879683300000124, "nonpadding_tokens": 2631, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2835414229999742, "nonpadding_tokens": 2912, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27716142700001, "nonpadding_tokens": 7632, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.2811747840000294, "nonpadding_tokens": 8042, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.283576987999993, "nonpadding_tokens": 2393, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2783123749999845, "nonpadding_tokens": 3799, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003388050000125986, "prefetch": {"data_wait_seconds": 0.0030345849998099084, "consumed_batches": 321, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 871, "tokens": 75164084, "loss_tokens": 54609, "loss": 2.909551024098592, "lr": 0.0002999182639161412, "optimizer_lrs": [0.019994550927742747, 0.0002999182639161412], "gradient_norm": 2.2870781421661377, "seconds": 0.25956157399997437, "nonpadding_tokens_per_second": 210389.38529477938, "ranks": [{"compute_seconds": 0.2520108060000439, "nonpadding_tokens": 2146, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.254386070999999, "nonpadding_tokens": 5237, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25123834200002193, "nonpadding_tokens": 11809, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2566935199999989, "nonpadding_tokens": 2304, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25557910000003403, "nonpadding_tokens": 4496, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.25424289200003614, "nonpadding_tokens": 1322, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.25717797099997597, "nonpadding_tokens": 14225, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.24925324800000226, "nonpadding_tokens": 13070, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033716900003355477, "prefetch": {"data_wait_seconds": 0.0030430069998175213, "consumed_batches": 322, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 872, "tokens": 75226218, "loss_tokens": 62134, "loss": 2.9821346303955965, "lr": 0.00029991807970439986, "optimizer_lrs": [0.01999453864695999, 0.00029991807970439986], "gradient_norm": 0.7353181838989258, "seconds": 0.29412385400001995, "nonpadding_tokens_per_second": 211251.14184038874, "ranks": [{"compute_seconds": 0.2914605720000054, "nonpadding_tokens": 7759, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2893602519999945, "nonpadding_tokens": 7341, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2863105910000172, "nonpadding_tokens": 7590, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29162788899998304, "nonpadding_tokens": 11207, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2853253289999884, "nonpadding_tokens": 12262, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.28835862400001133, "nonpadding_tokens": 2750, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2909757120000336, "nonpadding_tokens": 8519, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.28553004700000884, "nonpadding_tokens": 4706, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035045399999944493, "prefetch": {"data_wait_seconds": 0.003051468999800022, "consumed_batches": 323, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 873, "tokens": 75299699, "loss_tokens": 73481, "loss": 2.9979884255794014, "lr": 0.000299917861584122, "optimizer_lrs": [0.019994524105608133, 0.000299917861584122], "gradient_norm": 0.42152413725852966, "seconds": 0.3614527469999871, "nonpadding_tokens_per_second": 203293.51653814552, "ranks": [{"compute_seconds": 0.35896175900001026, "nonpadding_tokens": 20315, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35375727799998913, "nonpadding_tokens": 3783, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3561382050000361, "nonpadding_tokens": 17524, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3566793729999631, "nonpadding_tokens": 4676, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34904112999998915, "nonpadding_tokens": 8784, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18551406592}, {"compute_seconds": 0.3526830129999894, "nonpadding_tokens": 7422, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3493470029999912, "nonpadding_tokens": 5965, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3579747950000183, "nonpadding_tokens": 5012, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004289350000021841, "prefetch": {"data_wait_seconds": 0.0030605989998235827, "consumed_batches": 324, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 874, "tokens": 75363080, "loss_tokens": 63381, "loss": 2.993666526640476, "lr": 0.00029991767321171, "optimizer_lrs": [0.019994511547447334, 0.00029991767321171], "gradient_norm": 0.4855020344257355, "seconds": 0.2549754020000137, "nonpadding_tokens_per_second": 248576.9195884888, "ranks": [{"compute_seconds": 0.2476689869999973, "nonpadding_tokens": 8699, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.248122163000005, "nonpadding_tokens": 9934, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25111380099997405, "nonpadding_tokens": 2692, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2514737299999865, "nonpadding_tokens": 14609, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24610720999999103, "nonpadding_tokens": 17017, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24875204400001394, "nonpadding_tokens": 2939, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2458437340000046, "nonpadding_tokens": 2957, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2524151609999876, "nonpadding_tokens": 4534, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003888519999577511, "prefetch": {"data_wait_seconds": 0.0030695019998034923, "consumed_batches": 325, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 875, "tokens": 75430904, "loss_tokens": 67824, "loss": 2.8692154970954236, "lr": 0.0002999174713955173, "optimizer_lrs": [0.01999449809303449, 0.0002999174713955173], "gradient_norm": 0.862492561340332, "seconds": 0.2784405429999879, "nonpadding_tokens_per_second": 243585.21668305664, "ranks": [{"compute_seconds": 0.2726770129999636, "nonpadding_tokens": 4903, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27170427200002223, "nonpadding_tokens": 6683, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2698866830000384, "nonpadding_tokens": 20365, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27518898400001035, "nonpadding_tokens": 13018, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27536943400002656, "nonpadding_tokens": 5358, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27155951100002085, "nonpadding_tokens": 3947, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2686859870000262, "nonpadding_tokens": 2463, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.27597284600000194, "nonpadding_tokens": 11087, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035566599996172954, "prefetch": {"data_wait_seconds": 0.0030779309998507642, "consumed_batches": 326, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 876, "tokens": 75479940, "loss_tokens": 49036, "loss": 3.1047406497267316, "lr": 0.00029991732533081087, "optimizer_lrs": [0.019994488355387394, 0.00029991732533081087], "gradient_norm": 0.6511416435241699, "seconds": 0.2933685639999908, "nonpadding_tokens_per_second": 167148.1065708238, "ranks": [{"compute_seconds": 0.2873169990000406, "nonpadding_tokens": 10123, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2883447689999912, "nonpadding_tokens": 14575, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2894363209999824, "nonpadding_tokens": 1787, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29029389900000524, "nonpadding_tokens": 3878, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.284884892999969, "nonpadding_tokens": 3060, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2875458220000269, "nonpadding_tokens": 2138, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.28484528500001716, "nonpadding_tokens": 2518, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.29086083400000007, "nonpadding_tokens": 10957, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004106129999854602, "prefetch": {"data_wait_seconds": 0.0030874479998175275, "consumed_batches": 327, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 877, "tokens": 75545641, "loss_tokens": 65701, "loss": 3.3029108670339875, "lr": 0.0002999171294232965, "optimizer_lrs": [0.019994475294886432, 0.0002999171294232965], "gradient_norm": 0.7075068354606628, "seconds": 0.3049462650000123, "nonpadding_tokens_per_second": 215451.07299476958, "ranks": [{"compute_seconds": 0.3005882050000537, "nonpadding_tokens": 13525, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30224045400001387, "nonpadding_tokens": 4422, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29911588400000255, "nonpadding_tokens": 9229, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3025512010000284, "nonpadding_tokens": 10987, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3026538840000512, "nonpadding_tokens": 8369, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2999353719999931, "nonpadding_tokens": 9672, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.30257466500000874, "nonpadding_tokens": 3321, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2983322960000123, "nonpadding_tokens": 6176, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003552140000238069, "prefetch": {"data_wait_seconds": 0.0030958019998621467, "consumed_batches": 328, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 878, "tokens": 75587926, "loss_tokens": 42285, "loss": 2.8921475922312876, "lr": 0.00029991700321512453, "optimizer_lrs": [0.019994466881008303, 0.00029991700321512453], "gradient_norm": 0.5523350238800049, "seconds": 0.24235840599999392, "nonpadding_tokens_per_second": 174473.00755064818, "ranks": [{"compute_seconds": 0.23780553900002133, "nonpadding_tokens": 3501, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23895084800000177, "nonpadding_tokens": 3843, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2359430399999951, "nonpadding_tokens": 3678, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2365711949999536, "nonpadding_tokens": 4583, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23468408900004079, "nonpadding_tokens": 4087, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23757576699995298, "nonpadding_tokens": 3771, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.23941611300000432, "nonpadding_tokens": 3600, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.2400162229999978, "nonpadding_tokens": 15222, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003335629999696721, "prefetch": {"data_wait_seconds": 0.003103571999886867, "consumed_batches": 329, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 879, "tokens": 75653518, "loss_tokens": 65592, "loss": 3.2897261670630566, "lr": 0.0002999168072525187, "optimizer_lrs": [0.01999445381683458, 0.0002999168072525187], "gradient_norm": 0.5193648934364319, "seconds": 0.30559365400000615, "nonpadding_tokens_per_second": 214637.96496244872, "ranks": [{"compute_seconds": 0.3007280140000148, "nonpadding_tokens": 3847, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2987357700000075, "nonpadding_tokens": 1749, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30123841100004256, "nonpadding_tokens": 9554, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3020173699999873, "nonpadding_tokens": 11972, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29680670599998393, "nonpadding_tokens": 13014, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2994130989999917, "nonpadding_tokens": 19647, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.30216740599996683, "nonpadding_tokens": 1535, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3031731869999703, "nonpadding_tokens": 4274, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003855059999864352, "prefetch": {"data_wait_seconds": 0.0031133959999039007, "consumed_batches": 330, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 880, "tokens": 75703159, "loss_tokens": 49641, "loss": 3.239051388972825, "lr": 0.0002999166587915417, "optimizer_lrs": [0.019994443919436115, 0.0002999166587915417], "gradient_norm": 0.5165091156959534, "seconds": 0.3319882110000094, "nonpadding_tokens_per_second": 149526.39387546986, "ranks": [{"compute_seconds": 0.3289318019999996, "nonpadding_tokens": 6420, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32826924900001586, "nonpadding_tokens": 12246, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32488460799999075, "nonpadding_tokens": 3302, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3257702090000407, "nonpadding_tokens": 2508, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32421937300000536, "nonpadding_tokens": 4260, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32635378600002696, "nonpadding_tokens": 13493, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3291471050000041, "nonpadding_tokens": 2337, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.3295083780000141, "nonpadding_tokens": 5075, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004547529999854305, "prefetch": {"data_wait_seconds": 0.00312305299992488, "consumed_batches": 331, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 881, "tokens": 75744743, "loss_tokens": 41584, "loss": 3.243167068463832, "lr": 0.00029991653432473626, "optimizer_lrs": [0.019994435621649084, 0.00029991653432473626], "gradient_norm": 0.6596599221229553, "seconds": 0.2582115670000462, "nonpadding_tokens_per_second": 161046.23229366235, "ranks": [{"compute_seconds": 0.2529733710000528, "nonpadding_tokens": 3290, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2527430180000465, "nonpadding_tokens": 4100, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25459352000001445, "nonpadding_tokens": 4909, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25064326099999334, "nonpadding_tokens": 2792, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2557576980000249, "nonpadding_tokens": 4171, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25304274600000554, "nonpadding_tokens": 12301, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.24891914099998758, "nonpadding_tokens": 7003, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17628659712}, {"compute_seconds": 0.25578159300005154, "nonpadding_tokens": 3018, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003480459999991581, "prefetch": {"data_wait_seconds": 0.003131335999924545, "consumed_batches": 332, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 882, "tokens": 75806600, "loss_tokens": 61857, "loss": 3.0752413328321775, "lr": 0.0002999163490061945, "optimizer_lrs": [0.019994423267079633, 0.0002999163490061945], "gradient_norm": 0.5107057094573975, "seconds": 0.27957872799998995, "nonpadding_tokens_per_second": 221250.73836090358, "ranks": [{"compute_seconds": 0.2728160300000013, "nonpadding_tokens": 5186, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.273658816999955, "nonpadding_tokens": 18668, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27620145899999216, "nonpadding_tokens": 6414, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27648387699997556, "nonpadding_tokens": 8364, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2771160969999755, "nonpadding_tokens": 3600, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2744130719999589, "nonpadding_tokens": 10505, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2710848050000436, "nonpadding_tokens": 4355, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2717645749999633, "nonpadding_tokens": 4765, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003416509999851769, "prefetch": {"data_wait_seconds": 0.0031396469999123156, "consumed_batches": 333, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 883, "tokens": 75864228, "loss_tokens": 57628, "loss": 2.9863464266502393, "lr": 0.00029991617617254157, "optimizer_lrs": [0.019994411744836106, 0.00029991617617254157], "gradient_norm": 0.5308962464332581, "seconds": 0.2701297430000409, "nonpadding_tokens_per_second": 213334.5234774509, "ranks": [{"compute_seconds": 0.2677713780000204, "nonpadding_tokens": 2519, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2634663270000033, "nonpadding_tokens": 6364, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26608126200000015, "nonpadding_tokens": 9148, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2621787019999715, "nonpadding_tokens": 11148, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2673087049999481, "nonpadding_tokens": 11082, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2643539019999821, "nonpadding_tokens": 4638, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.26062238399998705, "nonpadding_tokens": 3968, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2673745330000088, "nonpadding_tokens": 8761, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003792090000160897, "prefetch": {"data_wait_seconds": 0.003147849999905361, "consumed_batches": 334, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 884, "tokens": 75924320, "loss_tokens": 60092, "loss": 3.0457093394295414, "lr": 0.00029991599575913383, "optimizer_lrs": [0.01999439971727559, 0.00029991599575913383], "gradient_norm": 0.5636453628540039, "seconds": 0.3462620639999727, "nonpadding_tokens_per_second": 173544.8558985218, "ranks": [{"compute_seconds": 0.34340389100003677, "nonpadding_tokens": 12336, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34127204899999697, "nonpadding_tokens": 15767, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3430685339999968, "nonpadding_tokens": 2710, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3437210499999992, "nonpadding_tokens": 2041, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3370865589999994, "nonpadding_tokens": 3513, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34116960099999005, "nonpadding_tokens": 8829, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.343846520999989, "nonpadding_tokens": 10970, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3379425480000009, "nonpadding_tokens": 3926, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004359950000321078, "prefetch": {"data_wait_seconds": 0.003157739999906539, "consumed_batches": 335, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 885, "tokens": 76012413, "loss_tokens": 88093, "loss": 2.9163870142917143, "lr": 0.00029991573092825324, "optimizer_lrs": [0.019994382061883552, 0.00029991573092825324], "gradient_norm": 0.5280107259750366, "seconds": 0.2644976699999688, "nonpadding_tokens_per_second": 333057.75434623065, "ranks": [{"compute_seconds": 0.213907689999985, "nonpadding_tokens": 11421, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21743274899995413, "nonpadding_tokens": 18972, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21454573600004778, "nonpadding_tokens": 17922, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21965083799995, "nonpadding_tokens": 5651, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2200483830000053, "nonpadding_tokens": 17071, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21612263200000825, "nonpadding_tokens": 4338, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.21358414499997025, "nonpadding_tokens": 7722, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21392977499999688, "nonpadding_tokens": 4996, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.019639755000014247, "prefetch": {"data_wait_seconds": 0.0031676219998786337, "consumed_batches": 336, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 886, "tokens": 76080078, "loss_tokens": 67665, "loss": 2.8595174757998967, "lr": 0.00029991552722641296, "optimizer_lrs": [0.019994368481760866, 0.00029991552722641296], "gradient_norm": 0.5458288192749023, "seconds": 0.32715013099999624, "nonpadding_tokens_per_second": 206831.64574371141, "ranks": [{"compute_seconds": 0.32330814099998406, "nonpadding_tokens": 15942, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3226734849999957, "nonpadding_tokens": 11226, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3228961570000024, "nonpadding_tokens": 10823, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3203757289999771, "nonpadding_tokens": 5833, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32388090199998487, "nonpadding_tokens": 7046, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3214825740000151, "nonpadding_tokens": 5531, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3185327779999625, "nonpadding_tokens": 6771, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32475673900000857, "nonpadding_tokens": 4493, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037476100004596447, "prefetch": {"data_wait_seconds": 0.0031764319999325608, "consumed_batches": 337, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 887, "tokens": 76152585, "loss_tokens": 72507, "loss": 2.966540299557284, "lr": 0.0002999153086751485, "optimizer_lrs": [0.019994353911676567, 0.0002999153086751485], "gradient_norm": 0.4714094400405884, "seconds": 0.263669316000005, "nonpadding_tokens_per_second": 274992.1799774329, "ranks": [{"compute_seconds": 0.26101009199999226, "nonpadding_tokens": 12545, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2602887269999883, "nonpadding_tokens": 18931, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.259654495999996, "nonpadding_tokens": 9596, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2569213750000472, "nonpadding_tokens": 5741, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25480183100000886, "nonpadding_tokens": 5474, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25783177499999965, "nonpadding_tokens": 4014, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2554429069999742, "nonpadding_tokens": 2841, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2609173120000037, "nonpadding_tokens": 13365, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003810449999832599, "prefetch": {"data_wait_seconds": 0.003185667999915154, "consumed_batches": 338, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 888, "tokens": 76240002, "loss_tokens": 87417, "loss": 3.2468459367171145, "lr": 0.0002999150448067224, "optimizer_lrs": [0.019994336320448163, 0.0002999150448067224], "gradient_norm": 0.4477999210357666, "seconds": 0.27796905700000707, "nonpadding_tokens_per_second": 314484.6442386491, "ranks": [{"compute_seconds": 0.27466603499999565, "nonpadding_tokens": 1865, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27126881300000605, "nonpadding_tokens": 15025, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27424812900000006, "nonpadding_tokens": 9732, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2753093320000062, "nonpadding_tokens": 13297, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26745329999999967, "nonpadding_tokens": 11036, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2700488060000339, "nonpadding_tokens": 12344, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2674049229999582, "nonpadding_tokens": 5238, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27543164299999034, "nonpadding_tokens": 18880, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004124690000253395, "prefetch": {"data_wait_seconds": 0.003195613999949387, "consumed_batches": 339, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 889, "tokens": 76275742, "loss_tokens": 35740, "loss": 3.3512708974538334, "lr": 0.0002999149368072819, "optimizer_lrs": [0.01999432912048546, 0.0002999149368072819], "gradient_norm": 0.5616171956062317, "seconds": 0.35523578400000133, "nonpadding_tokens_per_second": 100609.23366886897, "ranks": [{"compute_seconds": 0.35140649900000653, "nonpadding_tokens": 6565, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.348687152000025, "nonpadding_tokens": 3071, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3513508010000237, "nonpadding_tokens": 3661, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34667307000000847, "nonpadding_tokens": 8068, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3527218889999517, "nonpadding_tokens": 4026, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3494684839999991, "nonpadding_tokens": 2057, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.35216901600000483, "nonpadding_tokens": 4683, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3527075260000174, "nonpadding_tokens": 3609, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041850999997450344, "prefetch": {"data_wait_seconds": 0.0032047969999808856, "consumed_batches": 340, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 890, "tokens": 76334217, "loss_tokens": 58475, "loss": 2.9292323108165883, "lr": 0.00029991475995912447, "optimizer_lrs": [0.0199943173306083, 0.00029991475995912447], "gradient_norm": 0.5050285458564758, "seconds": 0.21982666799999606, "nonpadding_tokens_per_second": 266005.0326560063, "ranks": [{"compute_seconds": 0.21674028699999326, "nonpadding_tokens": 2759, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21391543999999385, "nonpadding_tokens": 3838, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21599009200002683, "nonpadding_tokens": 7472, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21167294100001754, "nonpadding_tokens": 6812, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21725202300001456, "nonpadding_tokens": 7263, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21490366100005076, "nonpadding_tokens": 5660, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.21690973800002666, "nonpadding_tokens": 10059, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21111446899999464, "nonpadding_tokens": 14612, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040747200000623707, "prefetch": {"data_wait_seconds": 0.003214395000000536, "consumed_batches": 341, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 891, "tokens": 76372220, "loss_tokens": 38003, "loss": 3.1205455819277423, "lr": 0.0002999146449267845, "optimizer_lrs": [0.01999430966178563, 0.0002999146449267845], "gradient_norm": 0.49415794014930725, "seconds": 0.23359624599999051, "nonpadding_tokens_per_second": 162686.68975100544, "ranks": [{"compute_seconds": 0.21252720400002545, "nonpadding_tokens": 4296, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21470734799999036, "nonpadding_tokens": 3848, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21768175299996528, "nonpadding_tokens": 4078, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21283721599996852, "nonpadding_tokens": 3304, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21168477300000177, "nonpadding_tokens": 8506, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2145998579999855, "nonpadding_tokens": 2096, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.21800847100001874, "nonpadding_tokens": 4821, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21871600899999066, "nonpadding_tokens": 7054, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.011044283999979143, "prefetch": {"data_wait_seconds": 0.003223489000006907, "consumed_batches": 342, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 892, "tokens": 76420134, "loss_tokens": 47914, "loss": 3.3591578140000835, "lr": 0.00029991449978404984, "optimizer_lrs": [0.019994299985603325, 0.00029991449978404984], "gradient_norm": 0.4877336919307709, "seconds": 0.2900555449999729, "nonpadding_tokens_per_second": 165189.0502558897, "ranks": [{"compute_seconds": 0.2859895320000305, "nonpadding_tokens": 7695, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2833044379999592, "nonpadding_tokens": 5430, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28573789699998997, "nonpadding_tokens": 5553, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2874788500000136, "nonpadding_tokens": 7762, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2800182969999696, "nonpadding_tokens": 9573, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28394072000003234, "nonpadding_tokens": 3302, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.27998715499995797, "nonpadding_tokens": 3583, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2807420169999659, "nonpadding_tokens": 5016, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005251020000400786, "prefetch": {"data_wait_seconds": 0.0032305229999707308, "consumed_batches": 343, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 893, "tokens": 76523946, "loss_tokens": 103812, "loss": 2.9271188542750357, "lr": 0.00029991418489033246, "optimizer_lrs": [0.01999427899268883, 0.00029991418489033246], "gradient_norm": 0.854110598564148, "seconds": 0.2822413279999978, "nonpadding_tokens_per_second": 367812.89521143696, "ranks": [{"compute_seconds": 0.2793486679999546, "nonpadding_tokens": 11774, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2765344410000239, "nonpadding_tokens": 10261, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27826839400000836, "nonpadding_tokens": 11203, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27377030500002775, "nonpadding_tokens": 15563, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27912983999999597, "nonpadding_tokens": 17898, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27486534199999824, "nonpadding_tokens": 17889, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.27205032899996695, "nonpadding_tokens": 12771, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2796499930000209, "nonpadding_tokens": 6453, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039663100000097984, "prefetch": {"data_wait_seconds": 0.0032417209999522356, "consumed_batches": 344, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 894, "tokens": 76558497, "loss_tokens": 34551, "loss": 2.8706174387137855, "lr": 0.000299914079958189, "optimizer_lrs": [0.0199942719972126, 0.000299914079958189], "gradient_norm": 0.5490924715995789, "seconds": 0.29217228999999634, "nonpadding_tokens_per_second": 118255.56763100441, "ranks": [{"compute_seconds": 0.289480712999989, "nonpadding_tokens": 3235, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28669699099998525, "nonpadding_tokens": 5952, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2879314980000345, "nonpadding_tokens": 4230, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2837108950000129, "nonpadding_tokens": 5452, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2887970369999948, "nonpadding_tokens": 2200, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28654421400000274, "nonpadding_tokens": 5269, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.28831557699999166, "nonpadding_tokens": 4152, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2827913820000276, "nonpadding_tokens": 4061, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039185899998983587, "prefetch": {"data_wait_seconds": 0.0032509229999533318, "consumed_batches": 345, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 895, "tokens": 76608131, "loss_tokens": 49634, "loss": 2.881340159970988, "lr": 0.00029991392910647287, "optimizer_lrs": [0.019994261940431527, 0.00029991392910647287], "gradient_norm": 0.4733882248401642, "seconds": 0.3315435760000014, "nonpadding_tokens_per_second": 149705.81122042247, "ranks": [{"compute_seconds": 0.32815342400004965, "nonpadding_tokens": 2867, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3257863430000043, "nonpadding_tokens": 10061, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3225916869999992, "nonpadding_tokens": 5195, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32350640199996405, "nonpadding_tokens": 2816, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3288208769999983, "nonpadding_tokens": 11592, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3258981700000163, "nonpadding_tokens": 2893, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.32201116499999216, "nonpadding_tokens": 4584, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3289924679999672, "nonpadding_tokens": 9626, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004172629999743549, "prefetch": {"data_wait_seconds": 0.003260105999927987, "consumed_batches": 346, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 896, "tokens": 76663664, "loss_tokens": 55533, "loss": 3.0664177155925305, "lr": 0.0002999137601692502, "optimizer_lrs": [0.019994250677950014, 0.0002999137601692502], "gradient_norm": 0.5201807022094727, "seconds": 0.3187854180000045, "nonpadding_tokens_per_second": 174201.8199841224, "ranks": [{"compute_seconds": 0.3161137319999625, "nonpadding_tokens": 3800, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31193368300000657, "nonpadding_tokens": 15755, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30994552999999314, "nonpadding_tokens": 2419, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31532308200002035, "nonpadding_tokens": 7830, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31551124700001765, "nonpadding_tokens": 2306, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31234477999998944, "nonpadding_tokens": 13202, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3154835930000104, "nonpadding_tokens": 2809, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3162946580000039, "nonpadding_tokens": 7412, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043204399997875953, "prefetch": {"data_wait_seconds": 0.00326990899993973, "consumed_batches": 347, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 897, "tokens": 76708092, "loss_tokens": 44428, "loss": 3.0111781984334205, "lr": 0.0002999136248953832, "optimizer_lrs": [0.019994241659692213, 0.0002999136248953832], "gradient_norm": 0.5114681124687195, "seconds": 0.251098809000041, "nonpadding_tokens_per_second": 176934.3318549661, "ranks": [{"compute_seconds": 0.24497647999999117, "nonpadding_tokens": 5783, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24452949800001988, "nonpadding_tokens": 8013, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24743887099998574, "nonpadding_tokens": 4769, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2485924660000478, "nonpadding_tokens": 4948, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24303919300001553, "nonpadding_tokens": 3255, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24546192200000405, "nonpadding_tokens": 2029, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2426692610000032, "nonpadding_tokens": 11868, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24852463300004501, "nonpadding_tokens": 3763, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004095310000025165, "prefetch": {"data_wait_seconds": 0.0032799379999488565, "consumed_batches": 348, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 898, "tokens": 76781099, "loss_tokens": 73007, "loss": 2.957196741408358, "lr": 0.00029991340237434387, "optimizer_lrs": [0.01999422682495626, 0.00029991340237434387], "gradient_norm": 0.47431308031082153, "seconds": 0.30972126099999286, "nonpadding_tokens_per_second": 235718.3997129654, "ranks": [{"compute_seconds": 0.30535487699995656, "nonpadding_tokens": 7294, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30502141300001995, "nonpadding_tokens": 12683, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3062672750000388, "nonpadding_tokens": 4372, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30662241599998197, "nonpadding_tokens": 3245, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.301558636999971, "nonpadding_tokens": 11517, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30435652099998833, "nonpadding_tokens": 21279, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3015567520000104, "nonpadding_tokens": 9604, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3073337939999874, "nonpadding_tokens": 3013, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003860759999838592, "prefetch": {"data_wait_seconds": 0.003288513999976317, "consumed_batches": 349, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 899, "tokens": 76859964, "loss_tokens": 78865, "loss": 2.951071054967349, "lr": 0.0002999131616769303, "optimizer_lrs": [0.019994210778462022, 0.0002999131616769303], "gradient_norm": 0.9034673571586609, "seconds": 0.29319130799996174, "nonpadding_tokens_per_second": 268988.1925149374, "ranks": [{"compute_seconds": 0.22151024199996527, "nonpadding_tokens": 2896, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24386029999999437, "nonpadding_tokens": 5383, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24127089999996087, "nonpadding_tokens": 10490, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2466074669999898, "nonpadding_tokens": 3300, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24758130899999742, "nonpadding_tokens": 14724, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24482463599997573, "nonpadding_tokens": 5534, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2398528339999757, "nonpadding_tokens": 15562, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2472890270000221, "nonpadding_tokens": 20976, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.016207124999993994, "prefetch": {"data_wait_seconds": 0.003298197000006553, "consumed_batches": 350, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 900, "tokens": 76940423, "loss_tokens": 80459, "loss": 2.6490941815085947, "lr": 0.000299912915770471, "optimizer_lrs": [0.01999419438469807, 0.000299912915770471], "gradient_norm": 0.5771169662475586, "seconds": 0.28774099499997874, "nonpadding_tokens_per_second": 279622.99914896017, "ranks": [{"compute_seconds": 0.28520428899997796, "nonpadding_tokens": 4812, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2830787689999852, "nonpadding_tokens": 5390, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28392746700001226, "nonpadding_tokens": 10567, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2842822850000175, "nonpadding_tokens": 8546, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27951137200000176, "nonpadding_tokens": 18420, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28236152900001343, "nonpadding_tokens": 18084, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.27904888000000483, "nonpadding_tokens": 7193, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2801022749999902, "nonpadding_tokens": 7447, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047129300003234675, "prefetch": {"data_wait_seconds": 0.003308501999995883, "consumed_batches": 351, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 901, "tokens": 76997485, "loss_tokens": 57062, "loss": 3.31003146796467, "lr": 0.00029991274116149215, "optimizer_lrs": [0.01999418274409948, 0.00029991274116149215], "gradient_norm": 0.5240907669067383, "seconds": 0.2692017449999753, "nonpadding_tokens_per_second": 211967.42242516013, "ranks": [{"compute_seconds": 0.2660687470000198, "nonpadding_tokens": 7406, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26267451399996844, "nonpadding_tokens": 5023, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2653657700000167, "nonpadding_tokens": 10821, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26656778899996425, "nonpadding_tokens": 2709, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2598099240000238, "nonpadding_tokens": 4252, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26368563800002676, "nonpadding_tokens": 15318, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2662192440000126, "nonpadding_tokens": 4817, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26681776999998874, "nonpadding_tokens": 6716, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034967999999935273, "prefetch": {"data_wait_seconds": 0.003317337000012799, "consumed_batches": 352, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 902, "tokens": 77072040, "loss_tokens": 74555, "loss": 3.0219454178123533, "lr": 0.00029991251276073816, "optimizer_lrs": [0.019994167517382547, 0.00029991251276073816], "gradient_norm": 0.8230361342430115, "seconds": 0.3061698400000523, "nonpadding_tokens_per_second": 243508.63559907553, "ranks": [{"compute_seconds": 0.30092434299996285, "nonpadding_tokens": 9530, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2993909219999864, "nonpadding_tokens": 4791, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3026706780000268, "nonpadding_tokens": 6176, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.298348543999964, "nonpadding_tokens": 24007, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30306664800002636, "nonpadding_tokens": 16279, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3001915899999972, "nonpadding_tokens": 2329, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2967497729999877, "nonpadding_tokens": 4837, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3034985809999853, "nonpadding_tokens": 6606, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038130199999386605, "prefetch": {"data_wait_seconds": 0.0033267039999742565, "consumed_batches": 353, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 903, "tokens": 77143974, "loss_tokens": 71934, "loss": 2.719769163399783, "lr": 0.0002999122921065906, "optimizer_lrs": [0.01999415280710604, 0.0002999122921065906], "gradient_norm": 0.5588530898094177, "seconds": 0.280638166000017, "nonpadding_tokens_per_second": 256322.9407649266, "ranks": [{"compute_seconds": 0.2754838980000045, "nonpadding_tokens": 3075, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27515057899995554, "nonpadding_tokens": 3643, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2715242130000206, "nonpadding_tokens": 10742, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27625796299997774, "nonpadding_tokens": 17367, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27798271199998226, "nonpadding_tokens": 5659, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.274212384000009, "nonpadding_tokens": 8834, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2764957489999915, "nonpadding_tokens": 9829, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2771383000000469, "nonpadding_tokens": 12785, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005752430000143249, "prefetch": {"data_wait_seconds": 0.003335264000000393, "consumed_batches": 354, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 904, "tokens": 77197292, "loss_tokens": 53318, "loss": 3.0449865664503544, "lr": 0.0002999121283768682, "optimizer_lrs": [0.019994141891791217, 0.0002999121283768682], "gradient_norm": 0.4784303903579712, "seconds": 0.366473821999989, "nonpadding_tokens_per_second": 145489.2458867133, "ranks": [{"compute_seconds": 0.36382930000002034, "nonpadding_tokens": 18955, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3632193199999847, "nonpadding_tokens": 3772, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36368867999999566, "nonpadding_tokens": 3654, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.36384917600003064, "nonpadding_tokens": 6056, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3590897830000017, "nonpadding_tokens": 4786, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3617481760000487, "nonpadding_tokens": 3057, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3640747030000284, "nonpadding_tokens": 4944, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35987015799997835, "nonpadding_tokens": 8094, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036115199998221215, "prefetch": {"data_wait_seconds": 0.0033447880000068153, "consumed_batches": 355, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 905, "tokens": 77243387, "loss_tokens": 46095, "loss": 2.8168437805076474, "lr": 0.0002999119867046284, "optimizer_lrs": [0.01999413244697523, 0.0002999119867046284], "gradient_norm": 0.5161644816398621, "seconds": 0.3386123710000106, "nonpadding_tokens_per_second": 136129.10793503924, "ranks": [{"compute_seconds": 0.3362369639999656, "nonpadding_tokens": 7289, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33279616000004353, "nonpadding_tokens": 3256, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3318077579999681, "nonpadding_tokens": 5524, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3324159680000207, "nonpadding_tokens": 5596, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3306252259999951, "nonpadding_tokens": 6560, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3335002149999582, "nonpadding_tokens": 10148, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.33088404499994795, "nonpadding_tokens": 2565, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3311362129999793, "nonpadding_tokens": 5157, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034511299998030154, "prefetch": {"data_wait_seconds": 0.003353725000025065, "consumed_batches": 356, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 906, "tokens": 77320908, "loss_tokens": 77521, "loss": 2.933825390861831, "lr": 0.0002999117481878338, "optimizer_lrs": [0.01999411654585559, 0.0002999117481878338], "gradient_norm": 0.4365432858467102, "seconds": 0.27724215500001037, "nonpadding_tokens_per_second": 279614.7649335546, "ranks": [{"compute_seconds": 0.27268548300003204, "nonpadding_tokens": 11753, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27029757299999346, "nonpadding_tokens": 3346, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2733867700000019, "nonpadding_tokens": 4950, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.273634659000038, "nonpadding_tokens": 13527, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27433507200004215, "nonpadding_tokens": 23070, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27012257999996336, "nonpadding_tokens": 6577, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2670034390000069, "nonpadding_tokens": 10900, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2748410300000046, "nonpadding_tokens": 3398, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038580099999308004, "prefetch": {"data_wait_seconds": 0.0033624800000353616, "consumed_batches": 357, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 907, "tokens": 77374001, "loss_tokens": 53093, "loss": 2.9930072584898197, "lr": 0.00029991158464500627, "optimizer_lrs": [0.01999410564300042, 0.00029991158464500627], "gradient_norm": 0.7557232975959778, "seconds": 0.2614710979999586, "nonpadding_tokens_per_second": 203054.94720494273, "ranks": [{"compute_seconds": 0.2577307159999691, "nonpadding_tokens": 2215, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25660893999997825, "nonpadding_tokens": 7220, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25821338399998695, "nonpadding_tokens": 5702, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2589900340000213, "nonpadding_tokens": 12335, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25287592699999095, "nonpadding_tokens": 9613, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25663444099996013, "nonpadding_tokens": 3838, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.25895875000003343, "nonpadding_tokens": 9382, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.253376693000007, "nonpadding_tokens": 2788, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039202100003876694, "prefetch": {"data_wait_seconds": 0.003372205000019335, "consumed_batches": 358, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 908, "tokens": 77454492, "loss_tokens": 80491, "loss": 2.9119802602154277, "lr": 0.00029991133641923853, "optimizer_lrs": [0.019994089094615905, 0.00029991133641923853], "gradient_norm": 0.4844948947429657, "seconds": 0.3014175350000414, "nonpadding_tokens_per_second": 267041.5309447373, "ranks": [{"compute_seconds": 0.29871745999997756, "nonpadding_tokens": 4240, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2965579840000032, "nonpadding_tokens": 21921, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29706944999998086, "nonpadding_tokens": 6964, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2943102090000025, "nonpadding_tokens": 4996, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29219455300000163, "nonpadding_tokens": 10547, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29594847399999935, "nonpadding_tokens": 11055, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2981269239999733, "nonpadding_tokens": 15062, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2989436610000098, "nonpadding_tokens": 5706, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041762300003256314, "prefetch": {"data_wait_seconds": 0.003381305000019097, "consumed_batches": 359, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 909, "tokens": 77516929, "loss_tokens": 62437, "loss": 3.0087908611880776, "lr": 0.00029991114363054304, "optimizer_lrs": [0.019994076242036203, 0.00029991114363054304], "gradient_norm": 0.4931074380874634, "seconds": 0.3732675150000091, "nonpadding_tokens_per_second": 167271.45409371742, "ranks": [{"compute_seconds": 0.36944672399999945, "nonpadding_tokens": 6280, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36706889599997794, "nonpadding_tokens": 5165, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36946154200001047, "nonpadding_tokens": 6451, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3702886680000006, "nonpadding_tokens": 3097, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3634909229999721, "nonpadding_tokens": 4469, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36649328499999, "nonpadding_tokens": 4873, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3700928959999601, "nonpadding_tokens": 13807, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37067333500004906, "nonpadding_tokens": 18295, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000450842000020657, "prefetch": {"data_wait_seconds": 0.003391415000024267, "consumed_batches": 360, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 910, "tokens": 77557916, "loss_tokens": 40987, "loss": 2.8549717364652207, "lr": 0.00029991101695989365, "optimizer_lrs": [0.019994067797326244, 0.00029991101695989365], "gradient_norm": 0.5505353212356567, "seconds": 0.23974193399999422, "nonpadding_tokens_per_second": 170962.9989053187, "ranks": [{"compute_seconds": 0.23506482099998038, "nonpadding_tokens": 5444, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23499359900000627, "nonpadding_tokens": 5142, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23581990399998176, "nonpadding_tokens": 2391, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23640872499998977, "nonpadding_tokens": 6469, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2371031129999892, "nonpadding_tokens": 4637, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2344954579999694, "nonpadding_tokens": 5003, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.23079013599999598, "nonpadding_tokens": 3561, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2316546179999932, "nonpadding_tokens": 8340, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040795600000365084, "prefetch": {"data_wait_seconds": 0.003401667999995084, "consumed_batches": 361, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 911, "tokens": 77617717, "loss_tokens": 59801, "loss": 3.133796362100968, "lr": 0.00029991083198262875, "optimizer_lrs": [0.019994055465508583, 0.00029991083198262875], "gradient_norm": 0.5369696617126465, "seconds": 0.35041421699997954, "nonpadding_tokens_per_second": 170658.03012211542, "ranks": [{"compute_seconds": 0.34797926599998164, "nonpadding_tokens": 17547, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34434397600000466, "nonpadding_tokens": 5696, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34157274400001825, "nonpadding_tokens": 5806, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3465384849999964, "nonpadding_tokens": 4668, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3473009990000264, "nonpadding_tokens": 2716, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34448203299996294, "nonpadding_tokens": 10699, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.34690738000000465, "nonpadding_tokens": 6408, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3475362180000161, "nonpadding_tokens": 6261, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040236099999901853, "prefetch": {"data_wait_seconds": 0.0034096819999973604, "consumed_batches": 362, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 912, "tokens": 77675403, "loss_tokens": 57686, "loss": 2.8487544638213778, "lr": 0.00029991065336557477, "optimizer_lrs": [0.019994043557704986, 0.00029991065336557477], "gradient_norm": 0.46414023637771606, "seconds": 0.260231363999992, "nonpadding_tokens_per_second": 221671.97340594875, "ranks": [{"compute_seconds": 0.2538828559999615, "nonpadding_tokens": 9367, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25435807700000623, "nonpadding_tokens": 2781, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2563354160000131, "nonpadding_tokens": 4619, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2569750109999518, "nonpadding_tokens": 3035, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2504209410000158, "nonpadding_tokens": 14070, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2545530589999885, "nonpadding_tokens": 7126, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2504805799999872, "nonpadding_tokens": 9680, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25772075599996924, "nonpadding_tokens": 7008, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003894409999816162, "prefetch": {"data_wait_seconds": 0.0034201439999748118, "consumed_batches": 363, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 913, "tokens": 77756277, "loss_tokens": 80874, "loss": 2.7841169210747583, "lr": 0.00029991040264913373, "optimizer_lrs": [0.019994026843275583, 0.00029991040264913373], "gradient_norm": 0.43847358226776123, "seconds": 0.2706683650000059, "nonpadding_tokens_per_second": 298793.691682433, "ranks": [{"compute_seconds": 0.26506375600001775, "nonpadding_tokens": 10967, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26438249899996435, "nonpadding_tokens": 5500, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2614063970000302, "nonpadding_tokens": 11007, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26719837500002086, "nonpadding_tokens": 13337, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2680809809999687, "nonpadding_tokens": 25341, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2629774330000032, "nonpadding_tokens": 3614, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2606288850000169, "nonpadding_tokens": 7536, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.268203222000011, "nonpadding_tokens": 3572, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034385699996164476, "prefetch": {"data_wait_seconds": 0.003427818999981014, "consumed_batches": 364, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 914, "tokens": 77828124, "loss_tokens": 71847, "loss": 3.2139731391011455, "lr": 0.0002999101796226252, "optimizer_lrs": [0.019994011974841682, 0.0002999101796226252], "gradient_norm": 0.4971543550491333, "seconds": 0.31103760100000954, "nonpadding_tokens_per_second": 230991.36493146306, "ranks": [{"compute_seconds": 0.3061117069999568, "nonpadding_tokens": 4641, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30462460199998986, "nonpadding_tokens": 3763, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30219301099998575, "nonpadding_tokens": 6145, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3073087889999897, "nonpadding_tokens": 3766, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3080320189999952, "nonpadding_tokens": 29344, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30534610899997006, "nonpadding_tokens": 10123, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.30778455499995516, "nonpadding_tokens": 4282, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3084816179999734, "nonpadding_tokens": 9783, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044203199996672993, "prefetch": {"data_wait_seconds": 0.003438720999966449, "consumed_batches": 365, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 915, "tokens": 77887750, "loss_tokens": 59626, "loss": 3.124834384328984, "lr": 0.00029990999432194363, "optimizer_lrs": [0.01999399962146291, 0.00029990999432194363], "gradient_norm": 0.44918426871299744, "seconds": 0.26088942900003076, "nonpadding_tokens_per_second": 228548.93058926114, "ranks": [{"compute_seconds": 0.25835108899997294, "nonpadding_tokens": 6890, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25433118099999774, "nonpadding_tokens": 4271, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2568052989999501, "nonpadding_tokens": 2972, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2576791070000013, "nonpadding_tokens": 17865, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25739759499998627, "nonpadding_tokens": 3407, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2536771480000084, "nonpadding_tokens": 9809, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2570433019999996, "nonpadding_tokens": 9990, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25786872499998026, "nonpadding_tokens": 4422, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041710400000738446, "prefetch": {"data_wait_seconds": 0.0034490849999571083, "consumed_batches": 366, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 916, "tokens": 77937262, "loss_tokens": 49512, "loss": 3.0436292590685086, "lr": 0.0002999098403076449, "optimizer_lrs": [0.019993989353842997, 0.0002999098403076449], "gradient_norm": 0.45175814628601074, "seconds": 0.31228418300003113, "nonpadding_tokens_per_second": 158547.89545967834, "ranks": [{"compute_seconds": 0.309705914999995, "nonpadding_tokens": 4123, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3061229450000269, "nonpadding_tokens": 4328, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30840819100001227, "nonpadding_tokens": 1177, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3092437600000153, "nonpadding_tokens": 11160, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30965083799998183, "nonpadding_tokens": 5155, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30714899099996273, "nonpadding_tokens": 6080, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.30347574099999974, "nonpadding_tokens": 8901, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30379847699998663, "nonpadding_tokens": 8588, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039593899998635607, "prefetch": {"data_wait_seconds": 0.0034581749999489375, "consumed_batches": 367, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 917, "tokens": 77983961, "loss_tokens": 46699, "loss": 2.9710426079787577, "lr": 0.00029990969492298206, "optimizer_lrs": [0.01999397966153214, 0.00029990969492298206], "gradient_norm": 0.5919704437255859, "seconds": 0.2664954769999781, "nonpadding_tokens_per_second": 175233.74327288804, "ranks": [{"compute_seconds": 0.22457618800001455, "nonpadding_tokens": 5861, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22424227799996288, "nonpadding_tokens": 3944, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2268197989999976, "nonpadding_tokens": 12016, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22710373199998912, "nonpadding_tokens": 7015, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2276425019999806, "nonpadding_tokens": 4904, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22292189500001314, "nonpadding_tokens": 1749, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.22761995199999774, "nonpadding_tokens": 1668, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22032869999998184, "nonpadding_tokens": 9542, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.015199686000016754, "prefetch": {"data_wait_seconds": 0.003468559999944887, "consumed_batches": 368, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 918, "tokens": 78050183, "loss_tokens": 66222, "loss": 2.784232156609586, "lr": 0.00029990948855801826, "optimizer_lrs": [0.019993965903867886, 0.00029990948855801826], "gradient_norm": 0.5876500606536865, "seconds": 0.2857658469999933, "nonpadding_tokens_per_second": 231735.1800266095, "ranks": [{"compute_seconds": 0.2832744130000151, "nonpadding_tokens": 3193, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2790297259999761, "nonpadding_tokens": 13794, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2778896510000095, "nonpadding_tokens": 5736, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2822744559999819, "nonpadding_tokens": 5372, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28299872700000606, "nonpadding_tokens": 15606, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28037885499998083, "nonpadding_tokens": 10719, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2825542960000007, "nonpadding_tokens": 4030, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27611243900003046, "nonpadding_tokens": 7772, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003942980000033458, "prefetch": {"data_wait_seconds": 0.003477991999957908, "consumed_batches": 369, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 919, "tokens": 78081659, "loss_tokens": 31476, "loss": 2.912207664172703, "lr": 0.000299909390388071, "optimizer_lrs": [0.019993959359204734, 0.000299909390388071], "gradient_norm": 0.6148154735565186, "seconds": 0.2609423430000106, "nonpadding_tokens_per_second": 120624.3480384428, "ranks": [{"compute_seconds": 0.25363277500002823, "nonpadding_tokens": 5958, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2558780300000194, "nonpadding_tokens": 3520, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2571682980000105, "nonpadding_tokens": 6944, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25845574900000656, "nonpadding_tokens": 2087, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25823306300003424, "nonpadding_tokens": 3362, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2553877089999901, "nonpadding_tokens": 4840, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2516002360000016, "nonpadding_tokens": 1986, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2512508099999877, "nonpadding_tokens": 2779, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038345399997297136, "prefetch": {"data_wait_seconds": 0.003488188999952513, "consumed_batches": 370, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 920, "tokens": 78153314, "loss_tokens": 71655, "loss": 3.1854683465913056, "lr": 0.00029990916670616575, "optimizer_lrs": [0.019993944447077718, 0.00029990916670616575], "gradient_norm": 0.599255383014679, "seconds": 0.3678414979999616, "nonpadding_tokens_per_second": 194798.57598885562, "ranks": [{"compute_seconds": 0.3610399359999974, "nonpadding_tokens": 11818, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3612119910000047, "nonpadding_tokens": 4499, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35949743599996964, "nonpadding_tokens": 16484, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3646595589999606, "nonpadding_tokens": 11060, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.36491314599999214, "nonpadding_tokens": 4745, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36213397000000214, "nonpadding_tokens": 8738, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3644544930000393, "nonpadding_tokens": 11320, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36544068999995716, "nonpadding_tokens": 2991, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039639599998508857, "prefetch": {"data_wait_seconds": 0.0034978269999470513, "consumed_batches": 371, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 921, "tokens": 78226285, "loss_tokens": 72971, "loss": 2.9873128109111837, "lr": 0.00029990893863286603, "optimizer_lrs": [0.01999392924219107, 0.00029990893863286603], "gradient_norm": 0.4848243296146393, "seconds": 0.2944368419999819, "nonpadding_tokens_per_second": 247832.4366758576, "ranks": [{"compute_seconds": 0.29186770100000103, "nonpadding_tokens": 8150, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2909679009999877, "nonpadding_tokens": 12328, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29151684599997907, "nonpadding_tokens": 7302, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2875679300000229, "nonpadding_tokens": 8140, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2863136949999898, "nonpadding_tokens": 7093, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28952914199999213, "nonpadding_tokens": 10256, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2920084129999623, "nonpadding_tokens": 14786, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28676775600001747, "nonpadding_tokens": 4916, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034173699998518714, "prefetch": {"data_wait_seconds": 0.003506951999909802, "consumed_batches": 372, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 922, "tokens": 78275961, "loss_tokens": 49676, "loss": 2.9922790306184073, "lr": 0.0002999087832053156, "optimizer_lrs": [0.019993918880354372, 0.0002999087832053156], "gradient_norm": 0.5308480262756348, "seconds": 0.3265371810000488, "nonpadding_tokens_per_second": 152129.68963553518, "ranks": [{"compute_seconds": 0.3207720830000085, "nonpadding_tokens": 5287, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3201341630000343, "nonpadding_tokens": 6197, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31809136699996543, "nonpadding_tokens": 3351, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3236701100000232, "nonpadding_tokens": 4168, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32409389600002214, "nonpadding_tokens": 9133, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3200527010000087, "nonpadding_tokens": 3424, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3237646359999644, "nonpadding_tokens": 12330, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31753310299995974, "nonpadding_tokens": 5786, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041037900001583694, "prefetch": {"data_wait_seconds": 0.0035162739998924053, "consumed_batches": 373, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 923, "tokens": 78336077, "loss_tokens": 60116, "loss": 2.721427115909242, "lr": 0.00029990859493564495, "optimizer_lrs": [0.019993906329043, 0.00029990859493564495], "gradient_norm": 0.7874190807342529, "seconds": 0.3146549930000333, "nonpadding_tokens_per_second": 191053.69797832397, "ranks": [{"compute_seconds": 0.3093174780000254, "nonpadding_tokens": 4304, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3083055669999908, "nonpadding_tokens": 13122, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3064022440000258, "nonpadding_tokens": 7002, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3114895549999801, "nonpadding_tokens": 7170, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3120981479999614, "nonpadding_tokens": 10973, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30921590900004503, "nonpadding_tokens": 9594, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3052823449999664, "nonpadding_tokens": 3325, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30541123200003994, "nonpadding_tokens": 4626, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042663599998604695, "prefetch": {"data_wait_seconds": 0.0035270609999003, "consumed_batches": 374, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 924, "tokens": 78398558, "loss_tokens": 62481, "loss": 3.006931107056545, "lr": 0.00029990839905370473, "optimizer_lrs": [0.019993893270246983, 0.00029990839905370473], "gradient_norm": 0.5729906558990479, "seconds": 0.316238903999988, "nonpadding_tokens_per_second": 197575.31160682993, "ranks": [{"compute_seconds": 0.3129843370000458, "nonpadding_tokens": 9087, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.311234168999988, "nonpadding_tokens": 4729, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30844046700002536, "nonpadding_tokens": 9907, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3136070639999957, "nonpadding_tokens": 1465, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30733440799997425, "nonpadding_tokens": 11962, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31020463400000153, "nonpadding_tokens": 5019, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.31387002400003894, "nonpadding_tokens": 6897, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3073005990000297, "nonpadding_tokens": 13415, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003690899999924113, "prefetch": {"data_wait_seconds": 0.0035355509998709067, "consumed_batches": 375, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 925, "tokens": 78471530, "loss_tokens": 72972, "loss": 3.0673357589212302, "lr": 0.00029990817001647186, "optimizer_lrs": [0.019993878001098125, 0.00029990817001647186], "gradient_norm": 0.544084370136261, "seconds": 0.34078908199995794, "nonpadding_tokens_per_second": 214126.57815137695, "ranks": [{"compute_seconds": 0.33441020200001503, "nonpadding_tokens": 9657, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3338302099999737, "nonpadding_tokens": 4817, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33697640100001536, "nonpadding_tokens": 9227, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3316431559999842, "nonpadding_tokens": 9891, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33007864600000403, "nonpadding_tokens": 9386, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33565692600001285, "nonpadding_tokens": 10073, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.33754410000000235, "nonpadding_tokens": 17175, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33839764999999034, "nonpadding_tokens": 2746, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040409899997939647, "prefetch": {"data_wait_seconds": 0.003545220999868093, "consumed_batches": 376, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 926, "tokens": 78555610, "loss_tokens": 84080, "loss": 2.936220712416746, "lr": 0.00029990790576008193, "optimizer_lrs": [0.01999386038400546, 0.00029990790576008193], "gradient_norm": 0.5010608434677124, "seconds": 0.3956080080000106, "nonpadding_tokens_per_second": 212533.61484026822, "ranks": [{"compute_seconds": 0.38960336799999595, "nonpadding_tokens": 1940, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.39017982299998266, "nonpadding_tokens": 8277, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3870152980000512, "nonpadding_tokens": 9577, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3918431349999878, "nonpadding_tokens": 12853, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3862644609999961, "nonpadding_tokens": 22861, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.389922112000022, "nonpadding_tokens": 5776, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3855834490000234, "nonpadding_tokens": 13999, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3930574530000399, "nonpadding_tokens": 8797, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044531300000016927, "prefetch": {"data_wait_seconds": 0.003554846999875849, "consumed_batches": 377, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 927, "tokens": 78635523, "loss_tokens": 79913, "loss": 3.117311169646991, "lr": 0.0002999076542484273, "optimizer_lrs": [0.019993843616561823, 0.0002999076542484273], "gradient_norm": 0.5466061234474182, "seconds": 0.26158078700001397, "nonpadding_tokens_per_second": 305500.26596561825, "ranks": [{"compute_seconds": 0.2573395899999582, "nonpadding_tokens": 9543, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2565429770000378, "nonpadding_tokens": 4148, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25807884099998546, "nonpadding_tokens": 4344, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2588218060000145, "nonpadding_tokens": 4808, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2590284799999836, "nonpadding_tokens": 18693, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25540618599995923, "nonpadding_tokens": 8050, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.25192428900004415, "nonpadding_tokens": 18920, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25317770899999914, "nonpadding_tokens": 11407, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003771059999735371, "prefetch": {"data_wait_seconds": 0.003564774999873066, "consumed_batches": 378, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 928, "tokens": 78691971, "loss_tokens": 56448, "loss": 2.720339958900227, "lr": 0.0002999074763819721, "optimizer_lrs": [0.019993831758798142, 0.0002999074763819721], "gradient_norm": 0.4958237409591675, "seconds": 0.27805220399994823, "nonpadding_tokens_per_second": 203012.23722726008, "ranks": [{"compute_seconds": 0.27345084099999895, "nonpadding_tokens": 1820, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2725078010000175, "nonpadding_tokens": 13244, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27483916700003874, "nonpadding_tokens": 8539, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.270805949000021, "nonpadding_tokens": 4218, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2756307220000167, "nonpadding_tokens": 8833, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27197235200003433, "nonpadding_tokens": 11953, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.26929084199997533, "nonpadding_tokens": 4198, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26985712199996215, "nonpadding_tokens": 3643, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037385100000619786, "prefetch": {"data_wait_seconds": 0.003573155999859523, "consumed_batches": 379, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 929, "tokens": 78765869, "loss_tokens": 73898, "loss": 2.9041618176405315, "lr": 0.000299907243272366, "optimizer_lrs": [0.019993816218157733, 0.000299907243272366], "gradient_norm": 0.45187437534332275, "seconds": 0.40462310600003093, "nonpadding_tokens_per_second": 182634.1573286088, "ranks": [{"compute_seconds": 0.40214844100000846, "nonpadding_tokens": 3934, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.40025330800000347, "nonpadding_tokens": 6455, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39720365299996274, "nonpadding_tokens": 13883, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3978292649999844, "nonpadding_tokens": 19670, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.40168568600000754, "nonpadding_tokens": 15240, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39921512099999745, "nonpadding_tokens": 2281, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.40144785000001093, "nonpadding_tokens": 2530, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3969647030000374, "nonpadding_tokens": 9905, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004059409999968011, "prefetch": {"data_wait_seconds": 0.0035828529998553904, "consumed_batches": 380, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 930, "tokens": 78882713, "loss_tokens": 116844, "loss": 3.305475944421622, "lr": 0.00029990687409228066, "optimizer_lrs": [0.019993791606152046, 0.00029990687409228066], "gradient_norm": 3.2559969425201416, "seconds": 0.30001261699999304, "nonpadding_tokens_per_second": 389463.62045834464, "ranks": [{"compute_seconds": 0.29750138099996093, "nonpadding_tokens": 11296, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29362809099995957, "nonpadding_tokens": 17332, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29166282700003876, "nonpadding_tokens": 19330, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2926713830000267, "nonpadding_tokens": 19001, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2911707110000066, "nonpadding_tokens": 9025, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2946929320000322, "nonpadding_tokens": 9002, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.29684355099999493, "nonpadding_tokens": 12115, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29752407299997685, "nonpadding_tokens": 19743, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039435299999013296, "prefetch": {"data_wait_seconds": 0.0035920769998512014, "consumed_batches": 381, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 931, "tokens": 78976049, "loss_tokens": 93336, "loss": 3.2123554282377644, "lr": 0.00029990657866142245, "optimizer_lrs": [0.019993771910761498, 0.00029990657866142245], "gradient_norm": 4.612550735473633, "seconds": 0.3125426370000355, "nonpadding_tokens_per_second": 298634.4547927693, "ranks": [{"compute_seconds": 0.3100575660000118, "nonpadding_tokens": 22029, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3080525009999633, "nonpadding_tokens": 25266, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30493893300001673, "nonpadding_tokens": 16315, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30569399300003397, "nonpadding_tokens": 13063, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30386120600002187, "nonpadding_tokens": 7621, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3073936010000011, "nonpadding_tokens": 4631, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.30998926699999174, "nonpadding_tokens": 2308, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3049565740000162, "nonpadding_tokens": 2103, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043061200000238387, "prefetch": {"data_wait_seconds": 0.0036034229998591627, "consumed_batches": 382, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 932, "tokens": 79033375, "loss_tokens": 57326, "loss": 3.1894379295607576, "lr": 0.0002999063969790174, "optimizer_lrs": [0.01999375979860116, 0.0002999063969790174], "gradient_norm": 0.6693567633628845, "seconds": 0.32762369000005265, "nonpadding_tokens_per_second": 174975.13687117922, "ranks": [{"compute_seconds": 0.32193415599999753, "nonpadding_tokens": 10032, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32168969599996444, "nonpadding_tokens": 5397, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32021692900002563, "nonpadding_tokens": 5914, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32100002499998936, "nonpadding_tokens": 2772, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32518433100000266, "nonpadding_tokens": 12685, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32199567000003526, "nonpadding_tokens": 4341, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.324706056000025, "nonpadding_tokens": 10666, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3201436109999918, "nonpadding_tokens": 5519, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041745400000081645, "prefetch": {"data_wait_seconds": 0.0036134629998656465, "consumed_batches": 383, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 933, "tokens": 79129521, "loss_tokens": 96146, "loss": 2.9025310465333973, "lr": 0.0002999060918688878, "optimizer_lrs": [0.019993739457925853, 0.0002999060918688878], "gradient_norm": 0.8498022556304932, "seconds": 0.27504423000004863, "nonpadding_tokens_per_second": 349565.5953225523, "ranks": [{"compute_seconds": 0.2726297070000214, "nonpadding_tokens": 6293, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2676911279999672, "nonpadding_tokens": 11692, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2708388149999905, "nonpadding_tokens": 5936, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.268558980000023, "nonpadding_tokens": 10393, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2673513149999849, "nonpadding_tokens": 21206, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2690139110000018, "nonpadding_tokens": 17292, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.267231613999968, "nonpadding_tokens": 10483, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26744757900002014, "nonpadding_tokens": 12851, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034694499998977335, "prefetch": {"data_wait_seconds": 0.0036217979998838246, "consumed_batches": 384, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 934, "tokens": 79172564, "loss_tokens": 43043, "loss": 2.8135004530353367, "lr": 0.00029990595511522275, "optimizer_lrs": [0.01999373034101485, 0.00029990595511522275], "gradient_norm": 1.3485770225524902, "seconds": 0.33025617399999874, "nonpadding_tokens_per_second": 130332.15845345608, "ranks": [{"compute_seconds": 0.327813510999988, "nonpadding_tokens": 4525, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32479475399998137, "nonpadding_tokens": 9578, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32679305699997485, "nonpadding_tokens": 7802, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32416123199999447, "nonpadding_tokens": 11165, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32269626999999446, "nonpadding_tokens": 953, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32554598900003384, "nonpadding_tokens": 3182, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3224606210000047, "nonpadding_tokens": 3448, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32322953800002097, "nonpadding_tokens": 2390, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043288799997753813, "prefetch": {"data_wait_seconds": 0.0036318629999101404, "consumed_batches": 385, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 935, "tokens": 79232827, "loss_tokens": 60263, "loss": 2.8533267718168696, "lr": 0.0002999057634840796, "optimizer_lrs": [0.01999371756560531, 0.0002999057634840796], "gradient_norm": 0.6424858570098877, "seconds": 0.2895428570000149, "nonpadding_tokens_per_second": 208131.53750153433, "ranks": [{"compute_seconds": 0.2824537459999874, "nonpadding_tokens": 16813, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28334309800004576, "nonpadding_tokens": 3122, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28539271699997926, "nonpadding_tokens": 1706, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28630228399998714, "nonpadding_tokens": 3137, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2868030939999926, "nonpadding_tokens": 6035, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2839427979999982, "nonpadding_tokens": 17131, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.28673928699998896, "nonpadding_tokens": 7670, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.287148894999973, "nonpadding_tokens": 4649, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003612360000033732, "prefetch": {"data_wait_seconds": 0.003641720999908671, "consumed_batches": 386, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 936, "tokens": 79305279, "loss_tokens": 72452, "loss": 2.7119030099238115, "lr": 0.00029990553283485344, "optimizer_lrs": [0.019993702188990232, 0.00029990553283485344], "gradient_norm": 1.3178333044052124, "seconds": 0.35563852699999643, "nonpadding_tokens_per_second": 203723.70960810082, "ranks": [{"compute_seconds": 0.3488574319999884, "nonpadding_tokens": 15285, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3503723169999944, "nonpadding_tokens": 11974, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34733118999997714, "nonpadding_tokens": 4525, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.351970381000001, "nonpadding_tokens": 11375, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3525121140000351, "nonpadding_tokens": 9051, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34954361500001596, "nonpadding_tokens": 7056, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3518821939999839, "nonpadding_tokens": 6615, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3530108350000205, "nonpadding_tokens": 6571, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038927600002125473, "prefetch": {"data_wait_seconds": 0.003650510999889889, "consumed_batches": 387, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 937, "tokens": 79352942, "loss_tokens": 47663, "loss": 2.916297429347712, "lr": 0.0002999053809471409, "optimizer_lrs": [0.019993692063142728, 0.0002999053809471409], "gradient_norm": 0.5036365389823914, "seconds": 0.2258834720000209, "nonpadding_tokens_per_second": 211007.0275526648, "ranks": [{"compute_seconds": 0.17957697499997494, "nonpadding_tokens": 5094, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.18905377199996565, "nonpadding_tokens": 8137, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19226088700003174, "nonpadding_tokens": 6227, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.18852120399998284, "nonpadding_tokens": 5462, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.1867377159999819, "nonpadding_tokens": 6803, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19011764700002232, "nonpadding_tokens": 7925, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.19363625099998671, "nonpadding_tokens": 4025, "peak_allocated_bytes": 14988478976, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.19351758299995936, "nonpadding_tokens": 3990, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.011242067000011957, "prefetch": {"data_wait_seconds": 0.0036587839998816207, "consumed_batches": 388, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 938, "tokens": 79423696, "loss_tokens": 70754, "loss": 2.74709777892416, "lr": 0.00029990515525041576, "optimizer_lrs": [0.019993677016694387, 0.00029990515525041576], "gradient_norm": 0.553134024143219, "seconds": 0.3675806800000032, "nonpadding_tokens_per_second": 192485.63335809537, "ranks": [{"compute_seconds": 0.36290060600003926, "nonpadding_tokens": 1681, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3609459289999677, "nonpadding_tokens": 10080, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36359057000004213, "nonpadding_tokens": 4015, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.36449368299997786, "nonpadding_tokens": 23475, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3649913989999618, "nonpadding_tokens": 4184, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36126760900003774, "nonpadding_tokens": 2784, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.35821025600000667, "nonpadding_tokens": 14757, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35906425800004627, "nonpadding_tokens": 9778, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003884509999920738, "prefetch": {"data_wait_seconds": 0.0036699549998644443, "consumed_batches": 389, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 939, "tokens": 79483110, "loss_tokens": 59414, "loss": 3.1128064303026224, "lr": 0.00029990496551932013, "optimizer_lrs": [0.019993664367954675, 0.00029990496551932013], "gradient_norm": 0.6073061227798462, "seconds": 0.2579706349999924, "nonpadding_tokens_per_second": 230313.03543522212, "ranks": [{"compute_seconds": 0.2553463410000063, "nonpadding_tokens": 2408, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2524190309999881, "nonpadding_tokens": 20169, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2510736510000129, "nonpadding_tokens": 9890, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2554246430000262, "nonpadding_tokens": 2867, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2501999869999736, "nonpadding_tokens": 3747, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25364363699998194, "nonpadding_tokens": 3908, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.25030092399998694, "nonpadding_tokens": 7984, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2512192419999906, "nonpadding_tokens": 8441, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004532039999958215, "prefetch": {"data_wait_seconds": 0.003681807999853959, "consumed_batches": 390, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 940, "tokens": 79555932, "loss_tokens": 72822, "loss": 2.8386770567273625, "lr": 0.0002999047327129775, "optimizer_lrs": [0.019993648847531836, 0.0002999047327129775], "gradient_norm": 0.474563866853714, "seconds": 0.33794779499999095, "nonpadding_tokens_per_second": 215482.9860629863, "ranks": [{"compute_seconds": 0.3350827449999656, "nonpadding_tokens": 14934, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3325625100000025, "nonpadding_tokens": 9125, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33443518399997174, "nonpadding_tokens": 6604, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32956678199997214, "nonpadding_tokens": 5797, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3354802810000024, "nonpadding_tokens": 12996, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33257438999999067, "nonpadding_tokens": 4688, "peak_allocated_bytes": 14987941376, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3349205549999965, "nonpadding_tokens": 5656, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32894440899997335, "nonpadding_tokens": 13022, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000390425000034611, "prefetch": {"data_wait_seconds": 0.0036897369998456497, "consumed_batches": 391, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 941, "tokens": 79619884, "loss_tokens": 63952, "loss": 2.8728971044689766, "lr": 0.0002999045280285451, "optimizer_lrs": [0.019993635201903007, 0.0002999045280285451], "gradient_norm": 0.5348416566848755, "seconds": 0.25293606099995714, "nonpadding_tokens_per_second": 252838.6017682581, "ranks": [{"compute_seconds": 0.24730971999997564, "nonpadding_tokens": 5796, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2464388199999803, "nonpadding_tokens": 9236, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24844722299997102, "nonpadding_tokens": 15635, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24914608400001725, "nonpadding_tokens": 4622, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24227969999998322, "nonpadding_tokens": 4836, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24721071499999425, "nonpadding_tokens": 11077, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.24274215200000526, "nonpadding_tokens": 6540, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2504917599999885, "nonpadding_tokens": 6210, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036808200002269587, "prefetch": {"data_wait_seconds": 0.003698381999882372, "consumed_batches": 392, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 942, "tokens": 79695205, "loss_tokens": 75321, "loss": 3.1079463230705913, "lr": 0.0002999042866749593, "optimizer_lrs": [0.019993619111663957, 0.0002999042866749593], "gradient_norm": 0.583235502243042, "seconds": 0.3017430049999916, "nonpadding_tokens_per_second": 249619.70535158584, "ranks": [{"compute_seconds": 0.29793024200000673, "nonpadding_tokens": 9246, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29522976399999834, "nonpadding_tokens": 2666, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.296776723999983, "nonpadding_tokens": 9577, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2972210540000333, "nonpadding_tokens": 13583, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2982206309999924, "nonpadding_tokens": 2735, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29367933500003573, "nonpadding_tokens": 3987, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.29095634699996253, "nonpadding_tokens": 12973, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29914955599997484, "nonpadding_tokens": 20554, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039617899994937034, "prefetch": {"data_wait_seconds": 0.003706213999876695, "consumed_batches": 393, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 943, "tokens": 79777771, "loss_tokens": 82566, "loss": 3.075270843930916, "lr": 0.00029990402175604643, "optimizer_lrs": [0.019993601450403096, 0.00029990402175604643], "gradient_norm": 0.4948658347129822, "seconds": 0.3263800370000354, "nonpadding_tokens_per_second": 252975.03106781942, "ranks": [{"compute_seconds": 0.3238843729999985, "nonpadding_tokens": 13944, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32032410599998684, "nonpadding_tokens": 12242, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3171196110000096, "nonpadding_tokens": 21920, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31732791000001725, "nonpadding_tokens": 14003, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32348583300000655, "nonpadding_tokens": 1983, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3187522970000032, "nonpadding_tokens": 7958, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3160181320000106, "nonpadding_tokens": 6660, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31717423999998573, "nonpadding_tokens": 3856, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037410099997714497, "prefetch": {"data_wait_seconds": 0.003715828999887094, "consumed_batches": 394, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 944, "tokens": 79833783, "loss_tokens": 56012, "loss": 2.8056080616653576, "lr": 0.000299903841829182, "optimizer_lrs": [0.019993589455278803, 0.000299903841829182], "gradient_norm": 0.5012519359588623, "seconds": 0.26518132899997227, "nonpadding_tokens_per_second": 211221.50722762936, "ranks": [{"compute_seconds": 0.26279135400000087, "nonpadding_tokens": 6243, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2586250310000082, "nonpadding_tokens": 3247, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26165457199999764, "nonpadding_tokens": 3524, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26185275000000274, "nonpadding_tokens": 11931, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2563291420000269, "nonpadding_tokens": 9676, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2591048880000244, "nonpadding_tokens": 8632, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2565545150000048, "nonpadding_tokens": 2062, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2569902520000369, "nonpadding_tokens": 10697, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004082600000288039, "prefetch": {"data_wait_seconds": 0.0037253119998723605, "consumed_batches": 395, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 945, "tokens": 79881918, "loss_tokens": 48135, "loss": 3.0482805520930714, "lr": 0.00029990368707096833, "optimizer_lrs": [0.019993579138064557, 0.00029990368707096833], "gradient_norm": 0.4806468188762665, "seconds": 0.26999706200001583, "nonpadding_tokens_per_second": 178279.7177252143, "ranks": [{"compute_seconds": 0.26634668200000533, "nonpadding_tokens": 1674, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2639610829999697, "nonpadding_tokens": 12372, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2665930850000109, "nonpadding_tokens": 16110, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26184823300002336, "nonpadding_tokens": 4389, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2602138830000058, "nonpadding_tokens": 3372, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26272100199997794, "nonpadding_tokens": 2296, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.26684844200002544, "nonpadding_tokens": 3814, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26749611799999684, "nonpadding_tokens": 4108, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034805900003220813, "prefetch": {"data_wait_seconds": 0.0037340839998734054, "consumed_batches": 396, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 946, "tokens": 79931717, "loss_tokens": 49799, "loss": 3.177757710998213, "lr": 0.00029990352683193503, "optimizer_lrs": [0.019993568455462338, 0.00029990352683193503], "gradient_norm": 0.5031557679176331, "seconds": 0.2819698789999734, "nonpadding_tokens_per_second": 176611.0627724343, "ranks": [{"compute_seconds": 0.2774955529999943, "nonpadding_tokens": 9143, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27557501899997305, "nonpadding_tokens": 11149, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.278671691999989, "nonpadding_tokens": 4945, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.278896173000021, "nonpadding_tokens": 6486, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27258870600002183, "nonpadding_tokens": 8520, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27658658000001424, "nonpadding_tokens": 1616, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.27237932899998896, "nonpadding_tokens": 3334, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2795209579999778, "nonpadding_tokens": 4606, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003381350000495331, "prefetch": {"data_wait_seconds": 0.003743495999870561, "consumed_batches": 397, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 947, "tokens": 79987965, "loss_tokens": 56248, "loss": 3.1211654192149054, "lr": 0.000299903345681734, "optimizer_lrs": [0.01999355637878227, 0.000299903345681734], "gradient_norm": 0.5061192512512207, "seconds": 0.29403170199998385, "nonpadding_tokens_per_second": 191299.10012221432, "ranks": [{"compute_seconds": 0.2883350139999834, "nonpadding_tokens": 2322, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2893455999999901, "nonpadding_tokens": 6908, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28946081500004084, "nonpadding_tokens": 13390, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2898456369999849, "nonpadding_tokens": 5700, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.291645012999993, "nonpadding_tokens": 5531, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2880049440000221, "nonpadding_tokens": 8930, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.29012831599999345, "nonpadding_tokens": 11441, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2861528639999733, "nonpadding_tokens": 2026, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004161769999768694, "prefetch": {"data_wait_seconds": 0.003752945999863755, "consumed_batches": 398, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 948, "tokens": 80042834, "loss_tokens": 54869, "loss": 3.0783448418050265, "lr": 0.0002999031688090293, "optimizer_lrs": [0.01999354458726862, 0.0002999031688090293], "gradient_norm": 0.6495783925056458, "seconds": 0.25312263199998597, "nonpadding_tokens_per_second": 216768.44763530683, "ranks": [{"compute_seconds": 0.24946469100001423, "nonpadding_tokens": 3798, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24703240100001267, "nonpadding_tokens": 4881, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2499044560000243, "nonpadding_tokens": 4935, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24981404599998314, "nonpadding_tokens": 10959, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24991553199998862, "nonpadding_tokens": 2643, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24550234700001283, "nonpadding_tokens": 5941, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.24224512500001083, "nonpadding_tokens": 15440, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25059647599999835, "nonpadding_tokens": 6272, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044185699999843564, "prefetch": {"data_wait_seconds": 0.0037622919998767657, "consumed_batches": 399, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 949, "tokens": 80090961, "loss_tokens": 48127, "loss": 2.8465774798969394, "lr": 0.00029990301353640823, "optimizer_lrs": [0.01999353423576055, 0.00029990301353640823], "gradient_norm": 0.45242950320243835, "seconds": 0.29097918200000095, "nonpadding_tokens_per_second": 165396.71212629857, "ranks": [{"compute_seconds": 0.2829978780000033, "nonpadding_tokens": 5971, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2844611680000071, "nonpadding_tokens": 2330, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2870407739999905, "nonpadding_tokens": 11018, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28298467700000174, "nonpadding_tokens": 13022, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28151598399995237, "nonpadding_tokens": 4166, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28388415500000974, "nonpadding_tokens": 6364, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2876479319999703, "nonpadding_tokens": 3534, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28846210400001837, "nonpadding_tokens": 1722, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004301399999917521, "prefetch": {"data_wait_seconds": 0.003771233999898982, "consumed_batches": 400, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 950, "tokens": 80140410, "loss_tokens": 49449, "loss": 2.7158432804505654, "lr": 0.0002999028538690864, "optimizer_lrs": [0.019993523591272426, 0.0002999028538690864], "gradient_norm": 1.4144163131713867, "seconds": 0.26983814100003656, "nonpadding_tokens_per_second": 183254.3013257466, "ranks": [{"compute_seconds": 0.2663046760000043, "nonpadding_tokens": 3445, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.262514613999997, "nonpadding_tokens": 5876, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26016247299997985, "nonpadding_tokens": 5990, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26614453599995613, "nonpadding_tokens": 4602, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25869644699997707, "nonpadding_tokens": 9130, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2636340720000021, "nonpadding_tokens": 6930, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.266126445999987, "nonpadding_tokens": 4065, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2668162269999925, "nonpadding_tokens": 9411, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003835029999663675, "prefetch": {"data_wait_seconds": 0.0037794799998920325, "consumed_batches": 401, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 951, "tokens": 80201065, "loss_tokens": 60655, "loss": 3.1637091954496745, "lr": 0.0002999026578391159, "optimizer_lrs": [0.01999351052260773, 0.0002999026578391159], "gradient_norm": 0.42661845684051514, "seconds": 0.30878424899998436, "nonpadding_tokens_per_second": 196431.651538039, "ranks": [{"compute_seconds": 0.3055493559999718, "nonpadding_tokens": 8798, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.302160458000003, "nonpadding_tokens": 6844, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29976650000003247, "nonpadding_tokens": 17512, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3059925219999968, "nonpadding_tokens": 6181, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30627647099998967, "nonpadding_tokens": 7751, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3020451259999959, "nonpadding_tokens": 7438, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.29910398800001303, "nonpadding_tokens": 4361, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2991039130000104, "nonpadding_tokens": 1770, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003987599999959457, "prefetch": {"data_wait_seconds": 0.0037892589998733683, "consumed_batches": 402, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 952, "tokens": 80260733, "loss_tokens": 59668, "loss": 3.1070425625963667, "lr": 0.000299902464806298, "optimizer_lrs": [0.0199934976537532, 0.000299902464806298], "gradient_norm": 0.5284122824668884, "seconds": 0.32274326099997097, "nonpadding_tokens_per_second": 184877.60151870488, "ranks": [{"compute_seconds": 0.3167198969999845, "nonpadding_tokens": 2147, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31452225000003864, "nonpadding_tokens": 5970, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3172219819999782, "nonpadding_tokens": 8252, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3137167720000207, "nonpadding_tokens": 10952, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31162385400000403, "nonpadding_tokens": 8415, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3156396740000105, "nonpadding_tokens": 4074, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.31201237300001594, "nonpadding_tokens": 7192, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32026932200000147, "nonpadding_tokens": 12666, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038169099997276135, "prefetch": {"data_wait_seconds": 0.003799437999830957, "consumed_batches": 403, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 953, "tokens": 80319333, "loss_tokens": 58600, "loss": 2.959785356228669, "lr": 0.00029990227504254695, "optimizer_lrs": [0.019993485002836466, 0.00029990227504254695], "gradient_norm": 0.4895733892917633, "seconds": 0.3691291860000092, "nonpadding_tokens_per_second": 158752.00938459128, "ranks": [{"compute_seconds": 0.36265443199999936, "nonpadding_tokens": 10732, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3624735649999593, "nonpadding_tokens": 3193, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3617835470000159, "nonpadding_tokens": 7180, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3620670450000034, "nonpadding_tokens": 16767, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3665385720000245, "nonpadding_tokens": 1569, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36344321899997567, "nonpadding_tokens": 7534, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3667027700000176, "nonpadding_tokens": 3357, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3610641550000082, "nonpadding_tokens": 8268, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034908300000324743, "prefetch": {"data_wait_seconds": 0.003809064999828138, "consumed_batches": 404, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 954, "tokens": 80387751, "loss_tokens": 68418, "loss": 2.879416326843813, "lr": 0.0002999020532519981, "optimizer_lrs": [0.019993470216799872, 0.0002999020532519981], "gradient_norm": 0.5955486297607422, "seconds": 0.2813321639999913, "nonpadding_tokens_per_second": 243192.95393470232, "ranks": [{"compute_seconds": 0.2745454960000302, "nonpadding_tokens": 5923, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2771658300000013, "nonpadding_tokens": 9742, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27410428799998954, "nonpadding_tokens": 10530, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27857975299997406, "nonpadding_tokens": 5182, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2787753090000251, "nonpadding_tokens": 5296, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27606890200002, "nonpadding_tokens": 23152, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2730483489999642, "nonpadding_tokens": 3829, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2735647410000297, "nonpadding_tokens": 4764, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046015300000590287, "prefetch": {"data_wait_seconds": 0.0038189679998055226, "consumed_batches": 405, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 955, "tokens": 80422441, "loss_tokens": 34690, "loss": 2.853323859541655, "lr": 0.0002999019407014544, "optimizer_lrs": [0.019993462713430294, 0.0002999019407014544], "gradient_norm": 0.6079636216163635, "seconds": 0.2569063290000031, "nonpadding_tokens_per_second": 135029.76020493283, "ranks": [{"compute_seconds": 0.25408221400005004, "nonpadding_tokens": 5734, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2538633910000385, "nonpadding_tokens": 3166, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25099004100002276, "nonpadding_tokens": 4767, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2514937949999876, "nonpadding_tokens": 5779, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25385824500000354, "nonpadding_tokens": 5157, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2524445740000374, "nonpadding_tokens": 2203, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2536159090000183, "nonpadding_tokens": 2801, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25448347000002514, "nonpadding_tokens": 5083, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004002370000080191, "prefetch": {"data_wait_seconds": 0.0038288779998225664, "consumed_batches": 406, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 956, "tokens": 80508375, "loss_tokens": 85934, "loss": 3.1308896944166453, "lr": 0.0002999016616132227, "optimizer_lrs": [0.01999344410754818, 0.0002999016616132227], "gradient_norm": 0.5231295228004456, "seconds": 0.386822895000023, "nonpadding_tokens_per_second": 222153.3448789139, "ranks": [{"compute_seconds": 0.3838854659999811, "nonpadding_tokens": 5628, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3833327280000276, "nonpadding_tokens": 18411, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38326351099999556, "nonpadding_tokens": 10551, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.38114073799999915, "nonpadding_tokens": 9601, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.38440829499995743, "nonpadding_tokens": 11499, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38256795099999863, "nonpadding_tokens": 8310, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3796902259999797, "nonpadding_tokens": 6782, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3802027429999839, "nonpadding_tokens": 15152, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036906200000430545, "prefetch": {"data_wait_seconds": 0.0038373589998315083, "consumed_batches": 407, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 957, "tokens": 80592019, "loss_tokens": 83644, "loss": 3.0655615241380136, "lr": 0.00029990138958149054, "optimizer_lrs": [0.019993425972099372, 0.00029990138958149054], "gradient_norm": 0.4637351632118225, "seconds": 0.33908707300003016, "nonpadding_tokens_per_second": 246674.10426463696, "ranks": [{"compute_seconds": 0.33528200799997876, "nonpadding_tokens": 13265, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3341255680000472, "nonpadding_tokens": 11165, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3352289509999764, "nonpadding_tokens": 21661, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33200421799995183, "nonpadding_tokens": 8436, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3304509629999757, "nonpadding_tokens": 8602, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33267793599998186, "nonpadding_tokens": 2704, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3361734260000162, "nonpadding_tokens": 11606, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3366035239999974, "nonpadding_tokens": 6205, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004272170000376718, "prefetch": {"data_wait_seconds": 0.0038467959997916523, "consumed_batches": 408, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 958, "tokens": 80648595, "loss_tokens": 56576, "loss": 3.1825890174278846, "lr": 0.00029990120536888774, "optimizer_lrs": [0.019993413691259183, 0.00029990120536888774], "gradient_norm": 0.5467850565910339, "seconds": 0.35340655099997775, "nonpadding_tokens_per_second": 160087.581398579, "ranks": [{"compute_seconds": 0.3478839680000192, "nonpadding_tokens": 10893, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34634874599998966, "nonpadding_tokens": 11983, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.348930418000009, "nonpadding_tokens": 11398, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3496553689999473, "nonpadding_tokens": 8269, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3430665460000455, "nonpadding_tokens": 3527, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34579354799996054, "nonpadding_tokens": 2577, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3499552769999923, "nonpadding_tokens": 3382, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35088976200000843, "nonpadding_tokens": 4547, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040415600000187624, "prefetch": {"data_wait_seconds": 0.003856342999782214, "consumed_batches": 409, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 959, "tokens": 80738728, "loss_tokens": 90133, "loss": 3.0935350398855026, "lr": 0.00029990091153906257, "optimizer_lrs": [0.019993394102604173, 0.00029990091153906257], "gradient_norm": 0.6889609694480896, "seconds": 0.27167618900000434, "nonpadding_tokens_per_second": 331766.2851932841, "ranks": [{"compute_seconds": 0.2673271910000494, "nonpadding_tokens": 8626, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2670297579999783, "nonpadding_tokens": 13309, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2691174509999996, "nonpadding_tokens": 7762, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2671349139999961, "nonpadding_tokens": 10618, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26800580000002583, "nonpadding_tokens": 15143, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2659493559999646, "nonpadding_tokens": 12136, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2631107560000032, "nonpadding_tokens": 17819, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2636058710000384, "nonpadding_tokens": 4720, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035446699996555253, "prefetch": {"data_wait_seconds": 0.003865104999775326, "consumed_batches": 410, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 960, "tokens": 80837855, "loss_tokens": 99127, "loss": 3.0065657565547226, "lr": 0.0002999005878855826, "optimizer_lrs": [0.019993372525705506, 0.0002999005878855826], "gradient_norm": 0.38612741231918335, "seconds": 0.30771291700000347, "nonpadding_tokens_per_second": 322141.1729037, "ranks": [{"compute_seconds": 0.30512683399996376, "nonpadding_tokens": 18157, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30156513200000745, "nonpadding_tokens": 15305, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3041305520000037, "nonpadding_tokens": 19904, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30434605200002807, "nonpadding_tokens": 10374, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29901677100002644, "nonpadding_tokens": 4404, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3027741369999717, "nonpadding_tokens": 3942, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.30448159600001645, "nonpadding_tokens": 6794, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30533826499998895, "nonpadding_tokens": 20247, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042888699999821256, "prefetch": {"data_wait_seconds": 0.0038760399998096773, "consumed_batches": 411, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 961, "tokens": 80912984, "loss_tokens": 75129, "loss": 3.208095617205074, "lr": 0.00029990034223508356, "optimizer_lrs": [0.019993356149005572, 0.00029990034223508356], "gradient_norm": 0.5165103673934937, "seconds": 0.3392498479999517, "nonpadding_tokens_per_second": 221456.2525021697, "ranks": [{"compute_seconds": 0.336188261000018, "nonpadding_tokens": 12692, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3351846969999883, "nonpadding_tokens": 5424, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33247282299998915, "nonpadding_tokens": 4174, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3363037920000238, "nonpadding_tokens": 14205, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3308823719999623, "nonpadding_tokens": 8637, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3339762680000149, "nonpadding_tokens": 3025, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3367602959999658, "nonpadding_tokens": 13480, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.331839978000005, "nonpadding_tokens": 13492, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038031399998317283, "prefetch": {"data_wait_seconds": 0.003886140999796339, "consumed_batches": 412, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 962, "tokens": 80991739, "loss_tokens": 78755, "loss": 2.7846735921528794, "lr": 0.00029990008440328533, "optimizer_lrs": [0.019993338960219022, 0.00029990008440328533], "gradient_norm": 0.9033555388450623, "seconds": 0.35493024700002707, "nonpadding_tokens_per_second": 221888.6687332511, "ranks": [{"compute_seconds": 0.34920773800001825, "nonpadding_tokens": 11989, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35048091499999146, "nonpadding_tokens": 6802, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35152579000003925, "nonpadding_tokens": 2892, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3481440450000264, "nonpadding_tokens": 7372, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3526112320000152, "nonpadding_tokens": 13450, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3496022780000203, "nonpadding_tokens": 3239, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.34667161200002283, "nonpadding_tokens": 3572, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.347391722999987, "nonpadding_tokens": 29439, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003970360000380424, "prefetch": {"data_wait_seconds": 0.003894478999782791, "consumed_batches": 413, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 963, "tokens": 81061211, "loss_tokens": 69472, "loss": 2.8470998207911102, "lr": 0.0002998998566861919, "optimizer_lrs": [0.01999332377907946, 0.0002998998566861919], "gradient_norm": 0.4903714656829834, "seconds": 0.34959854099997756, "nonpadding_tokens_per_second": 198719.36479278517, "ranks": [{"compute_seconds": 0.3459333320000155, "nonpadding_tokens": 17601, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34408763199996883, "nonpadding_tokens": 4452, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34648146799997903, "nonpadding_tokens": 12040, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3450552529999982, "nonpadding_tokens": 3039, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.347192739000036, "nonpadding_tokens": 12765, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34483872800001336, "nonpadding_tokens": 3848, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3432134329999599, "nonpadding_tokens": 6599, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34333026500002006, "nonpadding_tokens": 9128, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003819830000111324, "prefetch": {"data_wait_seconds": 0.0039026779998181382, "consumed_batches": 414, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 964, "tokens": 81107775, "loss_tokens": 46564, "loss": 2.8563110047461557, "lr": 0.0002998997039124925, "optimizer_lrs": [0.019993313594166166, 0.0002998997039124925], "gradient_norm": 0.4986128807067871, "seconds": 0.2869210299999736, "nonpadding_tokens_per_second": 162288.55723822085, "ranks": [{"compute_seconds": 0.23736811699995997, "nonpadding_tokens": 3253, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24196038899998484, "nonpadding_tokens": 5912, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2443598419999944, "nonpadding_tokens": 10549, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24194717099999252, "nonpadding_tokens": 9001, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2453820230000474, "nonpadding_tokens": 3719, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24257615400000532, "nonpadding_tokens": 5944, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2401153800000202, "nonpadding_tokens": 4508, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2408940360000429, "nonpadding_tokens": 3678, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.013803601000006438, "prefetch": {"data_wait_seconds": 0.0039113999998363624, "consumed_batches": 415, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 965, "tokens": 81165291, "loss_tokens": 57516, "loss": 2.805372092113499, "lr": 0.0002998995150452501, "optimizer_lrs": [0.019993301003016675, 0.0002998995150452501], "gradient_norm": 0.6764770746231079, "seconds": 0.4156696649999958, "nonpadding_tokens_per_second": 138369.49107171575, "ranks": [{"compute_seconds": 0.410700195000004, "nonpadding_tokens": 14433, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.409492317999991, "nonpadding_tokens": 9483, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4085998699999891, "nonpadding_tokens": 2421, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.41234343500002524, "nonpadding_tokens": 7290, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4084023430000343, "nonpadding_tokens": 4997, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4103301389999956, "nonpadding_tokens": 3515, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.4126881390000108, "nonpadding_tokens": 4793, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4132788559999767, "nonpadding_tokens": 10584, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004199009999865666, "prefetch": {"data_wait_seconds": 0.003922785999819212, "consumed_batches": 416, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 966, "tokens": 81229048, "loss_tokens": 63757, "loss": 2.7314922283043432, "lr": 0.0002998993054766785, "optimizer_lrs": [0.019993287031778568, 0.0002998993054766785], "gradient_norm": 0.467745840549469, "seconds": 0.3961742299999855, "nonpadding_tokens_per_second": 160931.71935994507, "ranks": [{"compute_seconds": 0.39182969999995976, "nonpadding_tokens": 13254, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.39115914899997506, "nonpadding_tokens": 7057, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39058459699998593, "nonpadding_tokens": 6181, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3909456459999774, "nonpadding_tokens": 3861, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3867074020000132, "nonpadding_tokens": 5316, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3887250870000116, "nonpadding_tokens": 8275, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.39119117100000267, "nonpadding_tokens": 10276, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3936200210000038, "nonpadding_tokens": 9537, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004295019999744909, "prefetch": {"data_wait_seconds": 0.003932438999825649, "consumed_batches": 417, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 967, "tokens": 81299505, "loss_tokens": 70457, "loss": 2.9086016116212727, "lr": 0.0002998990736314493, "optimizer_lrs": [0.019993271575429954, 0.0002998990736314493], "gradient_norm": 0.44283080101013184, "seconds": 0.29978262700001324, "nonpadding_tokens_per_second": 235026.96171915554, "ranks": [{"compute_seconds": 0.2963241379999886, "nonpadding_tokens": 7598, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29460370600003216, "nonpadding_tokens": 4512, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2916088009999953, "nonpadding_tokens": 21027, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2964681500000097, "nonpadding_tokens": 5983, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29672556999997823, "nonpadding_tokens": 4193, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2941091339999957, "nonpadding_tokens": 5815, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2972410770000238, "nonpadding_tokens": 9951, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29132325199998377, "nonpadding_tokens": 11378, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004072380000366138, "prefetch": {"data_wait_seconds": 0.003942102999829444, "consumed_batches": 418, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 968, "tokens": 81385740, "loss_tokens": 86235, "loss": 2.9628805009566883, "lr": 0.00029989878950455965, "optimizer_lrs": [0.019993252633637312, 0.00029989878950455965], "gradient_norm": 0.4728012681007385, "seconds": 0.3095729209999831, "nonpadding_tokens_per_second": 278561.1859120091, "ranks": [{"compute_seconds": 0.3067151479999666, "nonpadding_tokens": 12226, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3046574720000308, "nonpadding_tokens": 14494, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30580531299995073, "nonpadding_tokens": 15604, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30592036799998823, "nonpadding_tokens": 22039, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30099788800004035, "nonpadding_tokens": 4076, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3041112139999882, "nonpadding_tokens": 9950, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3005158870000173, "nonpadding_tokens": 3006, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30210286800001995, "nonpadding_tokens": 4840, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043289599994977834, "prefetch": {"data_wait_seconds": 0.003952279999865027, "consumed_batches": 419, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 969, "tokens": 81447346, "loss_tokens": 61606, "loss": 3.0381106345161184, "lr": 0.00029989858628076894, "optimizer_lrs": [0.019993239085384598, 0.00029989858628076894], "gradient_norm": 0.4967705011367798, "seconds": 0.21872373500002595, "nonpadding_tokens_per_second": 281661.24723497743, "ranks": [{"compute_seconds": 0.2155555499999764, "nonpadding_tokens": 3091, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2128656620000129, "nonpadding_tokens": 11373, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21122047300002578, "nonpadding_tokens": 11448, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21190692400000444, "nonpadding_tokens": 11704, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21619696699997348, "nonpadding_tokens": 12634, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.212858902999983, "nonpadding_tokens": 4303, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2101066879999962, "nonpadding_tokens": 5422, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21080244600000242, "nonpadding_tokens": 1631, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046851799999103605, "prefetch": {"data_wait_seconds": 0.003962124999887351, "consumed_batches": 420, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 970, "tokens": 81535395, "loss_tokens": 88049, "loss": 3.0917166861633865, "lr": 0.00029989829547400436, "optimizer_lrs": [0.019993219698266958, 0.00029989829547400436], "gradient_norm": 0.5346208214759827, "seconds": 0.24732134900000347, "nonpadding_tokens_per_second": 356010.51165218564, "ranks": [{"compute_seconds": 0.2439892109999846, "nonpadding_tokens": 4513, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24134118900002477, "nonpadding_tokens": 20269, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24462461600001006, "nonpadding_tokens": 15817, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23998406399999794, "nonpadding_tokens": 10304, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23820706599997266, "nonpadding_tokens": 11147, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24219850899999074, "nonpadding_tokens": 15926, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.24443648999999823, "nonpadding_tokens": 2965, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24494762400001946, "nonpadding_tokens": 7108, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004086469999720066, "prefetch": {"data_wait_seconds": 0.003971961999880591, "consumed_batches": 421, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 971, "tokens": 81583824, "loss_tokens": 48429, "loss": 3.1635441832373163, "lr": 0.0002998981353461, "optimizer_lrs": [0.019993209023073333, 0.0002998981353461], "gradient_norm": 0.5359680652618408, "seconds": 0.35780676399997446, "nonpadding_tokens_per_second": 135349.5933352547, "ranks": [{"compute_seconds": 0.3411905240000124, "nonpadding_tokens": 3379, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.339533222, "nonpadding_tokens": 5800, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34181068499998446, "nonpadding_tokens": 8163, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33907923900000014, "nonpadding_tokens": 5732, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33713796699998966, "nonpadding_tokens": 11484, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34018379399998366, "nonpadding_tokens": 5192, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3427346279999597, "nonpadding_tokens": 3849, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3384632839999995, "nonpadding_tokens": 4830, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.005296676000000389, "prefetch": {"data_wait_seconds": 0.003981660999855308, "consumed_batches": 422, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 972, "tokens": 81656682, "loss_tokens": 72858, "loss": 2.7690085079881412, "lr": 0.000299897894207814, "optimizer_lrs": [0.019993192947187603, 0.000299897894207814], "gradient_norm": 0.5500081181526184, "seconds": 0.3267486080000026, "nonpadding_tokens_per_second": 222978.7617029402, "ranks": [{"compute_seconds": 0.32228678299998137, "nonpadding_tokens": 3299, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.320047355999975, "nonpadding_tokens": 11868, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3229602630000272, "nonpadding_tokens": 14563, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32369376100001546, "nonpadding_tokens": 6170, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3241055089999918, "nonpadding_tokens": 5388, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32126659200002905, "nonpadding_tokens": 2112, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.31889391499998965, "nonpadding_tokens": 12291, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3243884990000083, "nonpadding_tokens": 17167, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041157399999747213, "prefetch": {"data_wait_seconds": 0.003991525999893497, "consumed_batches": 423, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 973, "tokens": 81699344, "loss_tokens": 42662, "loss": 2.8827701980099385, "lr": 0.0002998977528770107, "optimizer_lrs": [0.01999318352513405, 0.0002998977528770107], "gradient_norm": 0.5336344838142395, "seconds": 0.22097356500000842, "nonpadding_tokens_per_second": 193063.81738466487, "ranks": [{"compute_seconds": 0.21820530700000518, "nonpadding_tokens": 5161, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21540579100002333, "nonpadding_tokens": 4903, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21738058099998625, "nonpadding_tokens": 5309, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21256464900000083, "nonpadding_tokens": 2429, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21152650600004108, "nonpadding_tokens": 3445, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21490468600001122, "nonpadding_tokens": 10811, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.21783769199998915, "nonpadding_tokens": 8652, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21850693599998294, "nonpadding_tokens": 1952, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040274999997791383, "prefetch": {"data_wait_seconds": 0.0039996329998785995, "consumed_batches": 424, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 974, "tokens": 81739960, "loss_tokens": 40616, "loss": 3.191279298798503, "lr": 0.00029989761823341133, "optimizer_lrs": [0.01999317454889409, 0.00029989761823341133], "gradient_norm": 0.4979791045188904, "seconds": 0.27791584900001, "nonpadding_tokens_per_second": 146144.95771343555, "ranks": [{"compute_seconds": 0.2747095770000101, "nonpadding_tokens": 5356, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27141519300005257, "nonpadding_tokens": 7311, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27394461400001546, "nonpadding_tokens": 2394, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2708586969999942, "nonpadding_tokens": 4387, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26845918200001506, "nonpadding_tokens": 2179, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27200007200002574, "nonpadding_tokens": 8494, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2747640709999928, "nonpadding_tokens": 4317, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2755202149999718, "nonpadding_tokens": 6178, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004228579999789872, "prefetch": {"data_wait_seconds": 0.004009472999896957, "consumed_batches": 425, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 975, "tokens": 81810688, "loss_tokens": 70728, "loss": 2.8952779486200657, "lr": 0.00029989738355599125, "optimizer_lrs": [0.019993158903732752, 0.00029989738355599125], "gradient_norm": 0.5416088104248047, "seconds": 0.30319255899996733, "nonpadding_tokens_per_second": 233277.49280287456, "ranks": [{"compute_seconds": 0.29995717799999966, "nonpadding_tokens": 5030, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2973422189999724, "nonpadding_tokens": 1864, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29782673900001555, "nonpadding_tokens": 12726, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30083581599996023, "nonpadding_tokens": 2434, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3008675890000063, "nonpadding_tokens": 10601, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29903328400001783, "nonpadding_tokens": 6293, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.29677924799995026, "nonpadding_tokens": 18902, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.297424215000035, "nonpadding_tokens": 12878, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003743959999837898, "prefetch": {"data_wait_seconds": 0.004019966999877056, "consumed_batches": 426, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 976, "tokens": 81888526, "loss_tokens": 77838, "loss": 2.8118684800483056, "lr": 0.00029989712497699564, "optimizer_lrs": [0.019993141665133045, 0.00029989712497699564], "gradient_norm": 1.6862696409225464, "seconds": 0.2772152470000151, "nonpadding_tokens_per_second": 280785.4215897287, "ranks": [{"compute_seconds": 0.24004832299999634, "nonpadding_tokens": 2586, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24258784299996705, "nonpadding_tokens": 6529, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2437280400000077, "nonpadding_tokens": 4154, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24462376299999278, "nonpadding_tokens": 9689, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23896034299997382, "nonpadding_tokens": 5885, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2418990969999868, "nonpadding_tokens": 18092, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.24499686100000417, "nonpadding_tokens": 19214, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23900853399999278, "nonpadding_tokens": 11689, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.02225431300001901, "prefetch": {"data_wait_seconds": 0.004029015999904573, "consumed_batches": 427, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 977, "tokens": 81986144, "loss_tokens": 97618, "loss": 2.376920752320269, "lr": 0.0002998968002288059, "optimizer_lrs": [0.01999312001525373, 0.0002998968002288059], "gradient_norm": 1.8027327060699463, "seconds": 0.39389743300000646, "nonpadding_tokens_per_second": 247825.93594611823, "ranks": [{"compute_seconds": 0.3914686749999987, "nonpadding_tokens": 18654, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3889348050000194, "nonpadding_tokens": 20958, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.386488087000032, "nonpadding_tokens": 17961, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3908212299999718, "nonpadding_tokens": 9943, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.39116077599999244, "nonpadding_tokens": 14775, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3889439459999835, "nonpadding_tokens": 3407, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.38556251099998917, "nonpadding_tokens": 8310, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3860349079999992, "nonpadding_tokens": 3610, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042187700000795303, "prefetch": {"data_wait_seconds": 0.0040391389998859495, "consumed_batches": 428, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 978, "tokens": 82060494, "loss_tokens": 74350, "loss": 2.507506514794889, "lr": 0.00029989655254364584, "optimizer_lrs": [0.019993103502909722, 0.00029989655254364584], "gradient_norm": 0.7509604096412659, "seconds": 0.33287215499996137, "nonpadding_tokens_per_second": 223359.02502871898, "ranks": [{"compute_seconds": 0.3303465629999778, "nonpadding_tokens": 1521, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32529877100000704, "nonpadding_tokens": 3879, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3240415559999974, "nonpadding_tokens": 2503, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32849327000002404, "nonpadding_tokens": 8930, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32353422199997794, "nonpadding_tokens": 14888, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32711559699998816, "nonpadding_tokens": 10605, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3233080090000158, "nonpadding_tokens": 18890, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3242886790000057, "nonpadding_tokens": 13134, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035454300001447336, "prefetch": {"data_wait_seconds": 0.00404782399989756, "consumed_batches": 429, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 979, "tokens": 82130881, "loss_tokens": 70387, "loss": 2.7593734016934945, "lr": 0.0002998963177871342, "optimizer_lrs": [0.019993087852475614, 0.0002998963177871342], "gradient_norm": 0.5440229773521423, "seconds": 0.372497524000039, "nonpadding_tokens_per_second": 188959.64527268277, "ranks": [{"compute_seconds": 0.37011604799999986, "nonpadding_tokens": 3672, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36713498199998185, "nonpadding_tokens": 7985, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36945909600001414, "nonpadding_tokens": 15072, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.36897010399997043, "nonpadding_tokens": 12802, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.36721789500001023, "nonpadding_tokens": 7445, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36993116399997916, "nonpadding_tokens": 5571, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.37014139500001875, "nonpadding_tokens": 11332, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3677940430000035, "nonpadding_tokens": 6508, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041494800001373733, "prefetch": {"data_wait_seconds": 0.004056346999917082, "consumed_batches": 430, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 980, "tokens": 82182852, "loss_tokens": 51971, "loss": 3.0484332608570166, "lr": 0.0002998961442814451, "optimizer_lrs": [0.019993076285429675, 0.0002998961442814451], "gradient_norm": 0.5219213962554932, "seconds": 0.2784652850000384, "nonpadding_tokens_per_second": 186633.67679742497, "ranks": [{"compute_seconds": 0.27438417199999776, "nonpadding_tokens": 4840, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27592811700003494, "nonpadding_tokens": 7164, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2748690800000304, "nonpadding_tokens": 3707, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27547030599998834, "nonpadding_tokens": 12517, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2721856609999804, "nonpadding_tokens": 7051, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.274405153000032, "nonpadding_tokens": 3365, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.27568023899999616, "nonpadding_tokens": 4057, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2760541889999786, "nonpadding_tokens": 9270, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034912699999267716, "prefetch": {"data_wait_seconds": 0.004065632999939339, "consumed_batches": 431, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 981, "tokens": 82265856, "loss_tokens": 83004, "loss": 3.014029632608067, "lr": 0.0002998958668710634, "optimizer_lrs": [0.019993057791404225, 0.0002998958668710634], "gradient_norm": 0.45746374130249023, "seconds": 0.26263995099998283, "nonpadding_tokens_per_second": 316037.22009529854, "ranks": [{"compute_seconds": 0.2595445750000067, "nonpadding_tokens": 9912, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25635457900000347, "nonpadding_tokens": 11886, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2589650360000064, "nonpadding_tokens": 6926, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.259262619000026, "nonpadding_tokens": 12525, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2558474919999867, "nonpadding_tokens": 17537, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2570989719999943, "nonpadding_tokens": 12150, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2557018420000077, "nonpadding_tokens": 7918, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26030489200002194, "nonpadding_tokens": 4150, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004256449999502365, "prefetch": {"data_wait_seconds": 0.004074005999896713, "consumed_batches": 432, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 982, "tokens": 82349582, "loss_tokens": 83726, "loss": 2.9677765867233594, "lr": 0.0002998955866729616, "optimizer_lrs": [0.019993039111530775, 0.0002998955866729616], "gradient_norm": 0.4360470771789551, "seconds": 0.2963118270000109, "nonpadding_tokens_per_second": 282560.4392766844, "ranks": [{"compute_seconds": 0.29325812799999085, "nonpadding_tokens": 9717, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29295752399997355, "nonpadding_tokens": 10655, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29196446999998216, "nonpadding_tokens": 11813, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2909023049999746, "nonpadding_tokens": 17631, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28882666399999835, "nonpadding_tokens": 5357, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29197933699998657, "nonpadding_tokens": 15478, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.29268929900001694, "nonpadding_tokens": 10574, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29374049199998353, "nonpadding_tokens": 2501, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039354400001911927, "prefetch": {"data_wait_seconds": 0.0040835129999123865, "consumed_batches": 433, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 983, "tokens": 82447873, "loss_tokens": 98291, "loss": 2.925601657323661, "lr": 0.0002998952572513008, "optimizer_lrs": [0.01999301715008672, 0.0002998952572513008], "gradient_norm": 0.6051002144813538, "seconds": 0.43317683399999396, "nonpadding_tokens_per_second": 226907.3327222327, "ranks": [{"compute_seconds": 0.42853112500000634, "nonpadding_tokens": 12643, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4271291150000138, "nonpadding_tokens": 5219, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4296465189999594, "nonpadding_tokens": 11748, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.42865727599996717, "nonpadding_tokens": 27475, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.430903082000043, "nonpadding_tokens": 13766, "peak_allocated_bytes": 14978869760, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4296312069999999, "nonpadding_tokens": 10068, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.43060115499997664, "nonpadding_tokens": 13467, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.42748796299997593, "nonpadding_tokens": 3905, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003503670000100101, "prefetch": {"data_wait_seconds": 0.0040917329998961804, "consumed_batches": 434, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 984, "tokens": 82524584, "loss_tokens": 76711, "loss": 2.9693553564677817, "lr": 0.0002998949997945359, "optimizer_lrs": [0.019992999986302395, 0.0002998949997945359], "gradient_norm": 0.46018487215042114, "seconds": 0.347829747999981, "nonpadding_tokens_per_second": 220541.80368725734, "ranks": [{"compute_seconds": 0.3435055560000251, "nonpadding_tokens": 10122, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3455171130000281, "nonpadding_tokens": 11974, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3433003150000218, "nonpadding_tokens": 2342, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3430035589999534, "nonpadding_tokens": 12526, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3416554690000453, "nonpadding_tokens": 16526, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3417295169999761, "nonpadding_tokens": 3913, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.34399660200000426, "nonpadding_tokens": 7160, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3430411319999962, "nonpadding_tokens": 12148, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036076499998216605, "prefetch": {"data_wait_seconds": 0.004101074999894649, "consumed_batches": 435, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 985, "tokens": 82564492, "loss_tokens": 39908, "loss": 3.148418119487321, "lr": 0.00029989486573074576, "optimizer_lrs": [0.019992991048716387, 0.00029989486573074576], "gradient_norm": 0.5040609836578369, "seconds": 0.353416870999979, "nonpadding_tokens_per_second": 112920.47232233679, "ranks": [{"compute_seconds": 0.350795775999984, "nonpadding_tokens": 3223, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3508257999999955, "nonpadding_tokens": 2562, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34974015800003144, "nonpadding_tokens": 13079, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3484718839999914, "nonpadding_tokens": 2408, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3462600260000386, "nonpadding_tokens": 5162, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34947876199998973, "nonpadding_tokens": 5979, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3501969639999629, "nonpadding_tokens": 3612, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.351099645999966, "nonpadding_tokens": 3883, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035614900002656213, "prefetch": {"data_wait_seconds": 0.00411050699990767, "consumed_batches": 436, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 986, "tokens": 82613265, "loss_tokens": 48773, "loss": 3.0911813400857033, "lr": 0.00029989470177047743, "optimizer_lrs": [0.01999298011803183, 0.00029989470177047743], "gradient_norm": 0.567777156829834, "seconds": 0.264669997999988, "nonpadding_tokens_per_second": 184278.53692734076, "ranks": [{"compute_seconds": 0.26230118200004426, "nonpadding_tokens": 13528, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26237642199998845, "nonpadding_tokens": 3809, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2596812879999675, "nonpadding_tokens": 9326, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2619441959999449, "nonpadding_tokens": 4765, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2622945219999906, "nonpadding_tokens": 1832, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26119046200000184, "nonpadding_tokens": 9115, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2583170780000046, "nonpadding_tokens": 2593, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25913438199995653, "nonpadding_tokens": 3805, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003827900000032969, "prefetch": {"data_wait_seconds": 0.0041195999998677735, "consumed_batches": 437, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 987, "tokens": 82704440, "loss_tokens": 91175, "loss": 3.139489991774061, "lr": 0.00029989439492485084, "optimizer_lrs": [0.019992959661656724, 0.00029989439492485084], "gradient_norm": 0.5436550378799438, "seconds": 0.30513171499995906, "nonpadding_tokens_per_second": 298805.386388669, "ranks": [{"compute_seconds": 0.30278523700002324, "nonpadding_tokens": 3174, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3018110760000354, "nonpadding_tokens": 11205, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30159740499999543, "nonpadding_tokens": 4294, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3015238730000078, "nonpadding_tokens": 7815, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2981780839999715, "nonpadding_tokens": 18127, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2997364659999562, "nonpadding_tokens": 13584, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.29831881100000146, "nonpadding_tokens": 6819, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3028260809999779, "nonpadding_tokens": 26157, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000358925000000454, "prefetch": {"data_wait_seconds": 0.004128632999837123, "consumed_batches": 438, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 988, "tokens": 82814149, "loss_tokens": 109709, "loss": 2.970559047115551, "lr": 0.0002998940251122756, "optimizer_lrs": [0.01999293500748504, 0.0002998940251122756], "gradient_norm": 0.6797899007797241, "seconds": 0.3798966659999792, "nonpadding_tokens_per_second": 288786.4248853556, "ranks": [{"compute_seconds": 0.37750637100003814, "nonpadding_tokens": 20084, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3736563530000012, "nonpadding_tokens": 13377, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3763879819999829, "nonpadding_tokens": 17811, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.37597249999998894, "nonpadding_tokens": 14223, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.37727568500002917, "nonpadding_tokens": 13693, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3773097539999526, "nonpadding_tokens": 3681, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3770287309999958, "nonpadding_tokens": 23963, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37504815099998723, "nonpadding_tokens": 2877, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034919699999136355, "prefetch": {"data_wait_seconds": 0.004137311999855342, "consumed_batches": 439, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 989, "tokens": 82893555, "loss_tokens": 79406, "loss": 3.1907717537087876, "lr": 0.00029989375704349335, "optimizer_lrs": [0.019992917136232892, 0.00029989375704349335], "gradient_norm": 0.5141221284866333, "seconds": 0.2466298769999753, "nonpadding_tokens_per_second": 321964.2363119208, "ranks": [{"compute_seconds": 0.24434752100000878, "nonpadding_tokens": 5355, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2410232619999988, "nonpadding_tokens": 2310, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24171568799999932, "nonpadding_tokens": 12953, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24421345700000074, "nonpadding_tokens": 6285, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2408823170000005, "nonpadding_tokens": 21324, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24171276999999236, "nonpadding_tokens": 3550, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2401236300000278, "nonpadding_tokens": 2284, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2411135279999712, "nonpadding_tokens": 25345, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000396337000040603, "prefetch": {"data_wait_seconds": 0.004146557999845868, "consumed_batches": 440, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 990, "tokens": 82981566, "loss_tokens": 88011, "loss": 3.149600618104555, "lr": 0.00029989345952937405, "optimizer_lrs": [0.01999289730195827, 0.00029989345952937405], "gradient_norm": 0.5341674089431763, "seconds": 0.3356807069999945, "nonpadding_tokens_per_second": 262186.649886916, "ranks": [{"compute_seconds": 0.33296975700000075, "nonpadding_tokens": 17856, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3293516090000139, "nonpadding_tokens": 20420, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33210827199997084, "nonpadding_tokens": 12032, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3307285700000193, "nonpadding_tokens": 5650, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33334571200003893, "nonpadding_tokens": 11188, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3312460870000109, "nonpadding_tokens": 8399, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3286318309999956, "nonpadding_tokens": 1600, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3294439050000051, "nonpadding_tokens": 10866, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039787599996543577, "prefetch": {"data_wait_seconds": 0.004156889999833879, "consumed_batches": 441, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 991, "tokens": 83036093, "loss_tokens": 54527, "loss": 2.7211212449795514, "lr": 0.000299893274996614, "optimizer_lrs": [0.019992884999774266, 0.000299893274996614], "gradient_norm": 0.5340220928192139, "seconds": 0.25953493000002936, "nonpadding_tokens_per_second": 210095.03422138141, "ranks": [{"compute_seconds": 0.24078551299999162, "nonpadding_tokens": 4159, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24046567800002094, "nonpadding_tokens": 14028, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23976401599998098, "nonpadding_tokens": 5917, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23821883899995555, "nonpadding_tokens": 3703, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2407381579999992, "nonpadding_tokens": 4601, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23811577199995781, "nonpadding_tokens": 7938, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2370424039999648, "nonpadding_tokens": 4610, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23737735999998222, "nonpadding_tokens": 9571, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.015499493999982406, "prefetch": {"data_wait_seconds": 0.0041657779998018896, "consumed_batches": 442, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 992, "tokens": 83139176, "loss_tokens": 103083, "loss": 3.0026459260983867, "lr": 0.0002998929257023385, "optimizer_lrs": [0.019992861713489237, 0.0002998929257023385], "gradient_norm": 0.4314189553260803, "seconds": 0.2955502949999982, "nonpadding_tokens_per_second": 348783.2756181165, "ranks": [{"compute_seconds": 0.292229151000015, "nonpadding_tokens": 17567, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.292944518000013, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2893422949999831, "nonpadding_tokens": 9337, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2901470009999798, "nonpadding_tokens": 2849, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2931562780000263, "nonpadding_tokens": 9500, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29018539800000553, "nonpadding_tokens": 11217, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2887463020000496, "nonpadding_tokens": 8479, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2931438699999944, "nonpadding_tokens": 11366, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000394093000011253, "prefetch": {"data_wait_seconds": 0.004174290999813479, "consumed_batches": 443, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 993, "tokens": 83183722, "loss_tokens": 44546, "loss": 2.879877331859202, "lr": 0.0002998927745827702, "optimizer_lrs": [0.01999285163885135, 0.0002998927745827702], "gradient_norm": 0.46713966131210327, "seconds": 0.2997073199999818, "nonpadding_tokens_per_second": 148631.6717256112, "ranks": [{"compute_seconds": 0.29420504500001243, "nonpadding_tokens": 5629, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29359125600001335, "nonpadding_tokens": 11496, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2961849490000077, "nonpadding_tokens": 6319, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2967012880000084, "nonpadding_tokens": 5807, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2910717869999644, "nonpadding_tokens": 7679, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29360400500002015, "nonpadding_tokens": 3698, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2967901020000454, "nonpadding_tokens": 1591, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29741585599998643, "nonpadding_tokens": 2327, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000354211999990639, "prefetch": {"data_wait_seconds": 0.0041842919998345, "consumed_batches": 444, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 994, "tokens": 83242330, "loss_tokens": 58608, "loss": 3.152347215823778, "lr": 0.0002998925755964592, "optimizer_lrs": [0.01999283837309728, 0.0002998925755964592], "gradient_norm": 0.4462420344352722, "seconds": 0.28997393299999885, "nonpadding_tokens_per_second": 202114.71904959207, "ranks": [{"compute_seconds": 0.2811291200000028, "nonpadding_tokens": 6581, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2832991459999903, "nonpadding_tokens": 12373, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2859874830000422, "nonpadding_tokens": 13015, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2867481160000125, "nonpadding_tokens": 3307, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2870656350000331, "nonpadding_tokens": 6170, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2843392670000071, "nonpadding_tokens": 7190, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2792071480000118, "nonpadding_tokens": 4177, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2875354710000124, "nonpadding_tokens": 5795, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034057499999562424, "prefetch": {"data_wait_seconds": 0.0041922439998671734, "consumed_batches": 445, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 995, "tokens": 83288177, "loss_tokens": 45847, "loss": 2.8278143539926277, "lr": 0.00029989241980784854, "optimizer_lrs": [0.019992827987189904, 0.00029989241980784854], "gradient_norm": 0.6198498010635376, "seconds": 0.26668117399998437, "nonpadding_tokens_per_second": 171916.8972910052, "ranks": [{"compute_seconds": 0.26356402999999773, "nonpadding_tokens": 7970, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2606232549999845, "nonpadding_tokens": 5276, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2633808670000235, "nonpadding_tokens": 5563, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2596839710000154, "nonpadding_tokens": 3558, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2642166499999803, "nonpadding_tokens": 3705, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26026156999995465, "nonpadding_tokens": 6591, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2571645929999704, "nonpadding_tokens": 9949, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2584997299999827, "nonpadding_tokens": 3235, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034297899998136927, "prefetch": {"data_wait_seconds": 0.004200419999861538, "consumed_batches": 446, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 996, "tokens": 83335644, "loss_tokens": 47467, "loss": 3.023675329702741, "lr": 0.00029989225839557225, "optimizer_lrs": [0.019992817226371484, 0.00029989225839557225], "gradient_norm": 0.46518373489379883, "seconds": 0.23415948199999548, "nonpadding_tokens_per_second": 202712.26940961936, "ranks": [{"compute_seconds": 0.20934196800004656, "nonpadding_tokens": 6960, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20929262599997855, "nonpadding_tokens": 3543, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20901497900001687, "nonpadding_tokens": 2823, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21227968800002373, "nonpadding_tokens": 6699, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21289704499997697, "nonpadding_tokens": 3885, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21041242800004056, "nonpadding_tokens": 5083, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.21231412199995248, "nonpadding_tokens": 8209, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21310993200000894, "nonpadding_tokens": 10265, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.012592931999961365, "prefetch": {"data_wait_seconds": 0.004209680999863963, "consumed_batches": 447, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 997, "tokens": 83411672, "loss_tokens": 76028, "loss": 2.684131586389225, "lr": 0.0002998919996091568, "optimizer_lrs": [0.019992799973943785, 0.0002998919996091568], "gradient_norm": 0.49437975883483887, "seconds": 0.27293311599999015, "nonpadding_tokens_per_second": 278559.08844715916, "ranks": [{"compute_seconds": 0.2698027980000006, "nonpadding_tokens": 6370, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27062380000000985, "nonpadding_tokens": 9888, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26679761200000485, "nonpadding_tokens": 21375, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2681768020000277, "nonpadding_tokens": 21484, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27018451599997206, "nonpadding_tokens": 2501, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26839104699996597, "nonpadding_tokens": 9326, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2661152440000478, "nonpadding_tokens": 3511, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2704604150000023, "nonpadding_tokens": 1573, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003624299999955838, "prefetch": {"data_wait_seconds": 0.004219847999891613, "consumed_batches": 448, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 998, "tokens": 83466583, "loss_tokens": 54911, "loss": 2.779964966035949, "lr": 0.00029989181250844127, "optimizer_lrs": [0.019992787500562752, 0.00029989181250844127], "gradient_norm": 0.5032191872596741, "seconds": 0.2551754429999846, "nonpadding_tokens_per_second": 215189.20219922301, "ranks": [{"compute_seconds": 0.2521483770000259, "nonpadding_tokens": 4199, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2516080390000184, "nonpadding_tokens": 7755, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25197147299996914, "nonpadding_tokens": 11212, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.248826207000036, "nonpadding_tokens": 2436, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25283431999997674, "nonpadding_tokens": 4913, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.250700280999979, "nonpadding_tokens": 4022, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.24768503600000713, "nonpadding_tokens": 3117, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24830879000001005, "nonpadding_tokens": 17257, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037702099996295146, "prefetch": {"data_wait_seconds": 0.004228687999912495, "consumed_batches": 449, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 999, "tokens": 83532164, "loss_tokens": 65581, "loss": 2.930912726246931, "lr": 0.00029989158883925957, "optimizer_lrs": [0.01999277258928397, 0.00029989158883925957], "gradient_norm": 0.7981637716293335, "seconds": 0.3551831340000149, "nonpadding_tokens_per_second": 184639.96097291392, "ranks": [{"compute_seconds": 0.3514628259999881, "nonpadding_tokens": 28212, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34940016600000945, "nonpadding_tokens": 9596, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35168667199997117, "nonpadding_tokens": 4632, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34820763900000884, "nonpadding_tokens": 5338, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35273751500000117, "nonpadding_tokens": 3644, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3501383010000154, "nonpadding_tokens": 3589, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.34718582599998626, "nonpadding_tokens": 2730, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3469492230000242, "nonpadding_tokens": 7840, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000349915000015244, "prefetch": {"data_wait_seconds": 0.004238236999881906, "consumed_batches": 450, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1000, "tokens": 83577218, "loss_tokens": 45054, "loss": 2.692546028099614, "lr": 0.00029989143504524364, "optimizer_lrs": [0.019992762336349577, 0.00029989143504524364], "gradient_norm": 0.5557429790496826, "seconds": 0.30628741600003195, "nonpadding_tokens_per_second": 147097.1304939126, "ranks": [{"compute_seconds": 0.30389016200001606, "nonpadding_tokens": 6164, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3010935810000319, "nonpadding_tokens": 4556, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3027555150000012, "nonpadding_tokens": 3000, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29843841000001703, "nonpadding_tokens": 2248, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.303617370999973, "nonpadding_tokens": 4211, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29979275099998404, "nonpadding_tokens": 5350, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3033522769999877, "nonpadding_tokens": 4355, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2973187049999524, "nonpadding_tokens": 15170, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034501599998293386, "prefetch": {"data_wait_seconds": 0.004247840999894947, "consumed_batches": 451, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1001, "tokens": 83628117, "loss_tokens": 50899, "loss": 2.8777999690563667, "lr": 0.00029989126116794837, "optimizer_lrs": [0.019992750744529893, 0.00029989126116794837], "gradient_norm": 0.5062742233276367, "seconds": 0.28897585699996853, "nonpadding_tokens_per_second": 176135.821616425, "ranks": [{"compute_seconds": 0.26563225699999293, "nonpadding_tokens": 2851, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2701823959999956, "nonpadding_tokens": 8587, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.269123454999999, "nonpadding_tokens": 2579, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2695847439999852, "nonpadding_tokens": 5031, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2661524170000007, "nonpadding_tokens": 13143, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2693663160000028, "nonpadding_tokens": 6036, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2659980899999823, "nonpadding_tokens": 9988, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27068842200003473, "nonpadding_tokens": 2684, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01369036299996651, "prefetch": {"data_wait_seconds": 0.004256956999896033, "consumed_batches": 452, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1002, "tokens": 83685955, "loss_tokens": 57838, "loss": 2.7686569059268993, "lr": 0.0002998910634173637, "optimizer_lrs": [0.019992737561157582, 0.0002998910634173637], "gradient_norm": 0.5192904472351074, "seconds": 0.24346923100000595, "nonpadding_tokens_per_second": 237557.73886679992, "ranks": [{"compute_seconds": 0.23990319399996451, "nonpadding_tokens": 4899, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23907650000001013, "nonpadding_tokens": 2687, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24034244200004196, "nonpadding_tokens": 1642, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2408748830000036, "nonpadding_tokens": 1738, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2355384459999641, "nonpadding_tokens": 13430, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2386550390000366, "nonpadding_tokens": 12054, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.24083836199997677, "nonpadding_tokens": 10537, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23630206400002862, "nonpadding_tokens": 10851, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044906799996624613, "prefetch": {"data_wait_seconds": 0.004267000999917059, "consumed_batches": 453, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1003, "tokens": 83739977, "loss_tokens": 54022, "loss": 2.955085428158898, "lr": 0.0002998908785516496, "optimizer_lrs": [0.01999272523677664, 0.0002998908785516496], "gradient_norm": 0.5736029148101807, "seconds": 0.4011510410000483, "nonpadding_tokens_per_second": 134667.48052136676, "ranks": [{"compute_seconds": 0.39769795799998064, "nonpadding_tokens": 12920, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3985935990000371, "nonpadding_tokens": 3892, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39559720099998685, "nonpadding_tokens": 11710, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.39788561799997524, "nonpadding_tokens": 7660, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3943618170000036, "nonpadding_tokens": 2770, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39719446300000527, "nonpadding_tokens": 4872, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3945975419999854, "nonpadding_tokens": 5666, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.398764529999994, "nonpadding_tokens": 4532, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003951119999783259, "prefetch": {"data_wait_seconds": 0.00427629799987983, "consumed_batches": 454, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1004, "tokens": 83798265, "loss_tokens": 58288, "loss": 2.883298234628054, "lr": 0.00029989067891178376, "optimizer_lrs": [0.01999271192745225, 0.00029989067891178376], "gradient_norm": 0.5325256586074829, "seconds": 0.5646300889999907, "nonpadding_tokens_per_second": 103232.18888889388, "ranks": [{"compute_seconds": 0.5620193699999732, "nonpadding_tokens": 5336, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5574676000000522, "nonpadding_tokens": 7561, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5604174789999661, "nonpadding_tokens": 5474, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.5577679610000246, "nonpadding_tokens": 9913, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.5616457880000212, "nonpadding_tokens": 5560, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5588256009999668, "nonpadding_tokens": 11282, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.5562154769999665, "nonpadding_tokens": 3620, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.5619729080000297, "nonpadding_tokens": 9542, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038731699999061675, "prefetch": {"data_wait_seconds": 0.004286181999887617, "consumed_batches": 455, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1005, "tokens": 83848136, "loss_tokens": 49871, "loss": 2.702968345832247, "lr": 0.000299890507955865, "optimizer_lrs": [0.019992700530391, 0.000299890507955865], "gradient_norm": 0.4938918352127075, "seconds": 0.35802860299997974, "nonpadding_tokens_per_second": 139293.34020277375, "ranks": [{"compute_seconds": 0.3551413109999544, "nonpadding_tokens": 3986, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3556906270000013, "nonpadding_tokens": 2268, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35404340599995976, "nonpadding_tokens": 5306, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.35221346999998104, "nonpadding_tokens": 5751, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3548141120000423, "nonpadding_tokens": 16329, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35215068899998414, "nonpadding_tokens": 5551, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3505984839999883, "nonpadding_tokens": 4671, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3554424260000246, "nonpadding_tokens": 6009, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038260700000591896, "prefetch": {"data_wait_seconds": 0.00429582099987158, "consumed_batches": 456, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1006, "tokens": 83903657, "loss_tokens": 55521, "loss": 3.125275796545451, "lr": 0.00029989031747490095, "optimizer_lrs": [0.019992687831660066, 0.00029989031747490095], "gradient_norm": 0.5516709685325623, "seconds": 0.3508646679999856, "nonpadding_tokens_per_second": 158240.49858449207, "ranks": [{"compute_seconds": 0.34816126199996233, "nonpadding_tokens": 17617, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34715134200001785, "nonpadding_tokens": 1719, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34413021500000696, "nonpadding_tokens": 2449, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3482891259999974, "nonpadding_tokens": 7093, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3429612410000118, "nonpadding_tokens": 9768, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3457313520000298, "nonpadding_tokens": 5983, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.34286032299996805, "nonpadding_tokens": 6936, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3433906589999651, "nonpadding_tokens": 3956, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003702409999846168, "prefetch": {"data_wait_seconds": 0.0043053439998317344, "consumed_batches": 457, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1007, "tokens": 84020585, "loss_tokens": 116928, "loss": 2.7932140932881775, "lr": 0.0002998899157781191, "optimizer_lrs": [0.019992661051874607, 0.0002998899157781191], "gradient_norm": 0.4734296500682831, "seconds": 0.2798364939999942, "nonpadding_tokens_per_second": 417844.00000381086, "ranks": [{"compute_seconds": 0.2753590469999949, "nonpadding_tokens": 12162, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2739832179999553, "nonpadding_tokens": 10761, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27425377599996636, "nonpadding_tokens": 14265, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2765172240000311, "nonpadding_tokens": 12379, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2774832179999862, "nonpadding_tokens": 5613, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27476814099998137, "nonpadding_tokens": 17214, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2773257009999952, "nonpadding_tokens": 18694, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27374877400001196, "nonpadding_tokens": 25840, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003659540000171546, "prefetch": {"data_wait_seconds": 0.004313081999853807, "consumed_batches": 458, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1008, "tokens": 84092081, "loss_tokens": 71496, "loss": 2.7764590851236433, "lr": 0.00029988966979775205, "optimizer_lrs": [0.019992644653183473, 0.00029988966979775205], "gradient_norm": 0.4595464766025543, "seconds": 0.3931019130000095, "nonpadding_tokens_per_second": 181876.499797136, "ranks": [{"compute_seconds": 0.38940077399996653, "nonpadding_tokens": 18053, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.390204309000012, "nonpadding_tokens": 12874, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38737610999999106, "nonpadding_tokens": 5054, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.38979943399999684, "nonpadding_tokens": 3695, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3865557319999766, "nonpadding_tokens": 5024, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3872236990000033, "nonpadding_tokens": 11174, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3900892990000102, "nonpadding_tokens": 1527, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.39032539199996563, "nonpadding_tokens": 14095, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035235400002875394, "prefetch": {"data_wait_seconds": 0.004321113999836257, "consumed_batches": 459, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1009, "tokens": 84189734, "loss_tokens": 97653, "loss": 2.869744465095798, "lr": 0.00029988933338147153, "optimizer_lrs": [0.019992622225431435, 0.00029988933338147153], "gradient_norm": 0.6942232847213745, "seconds": 0.28927131499995085, "nonpadding_tokens_per_second": 337582.72921052197, "ranks": [{"compute_seconds": 0.28533639500000163, "nonpadding_tokens": 13484, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28685279600000513, "nonpadding_tokens": 12647, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28390432699995927, "nonpadding_tokens": 25780, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28615666099994996, "nonpadding_tokens": 20421, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2870026300000177, "nonpadding_tokens": 8648, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28546758200002387, "nonpadding_tokens": 1589, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.28653222100001585, "nonpadding_tokens": 10442, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2833631350000019, "nonpadding_tokens": 4642, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003468599999791877, "prefetch": {"data_wait_seconds": 0.004330301999871722, "consumed_batches": 460, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1010, "tokens": 84247160, "loss_tokens": 57426, "loss": 2.6703613672378363, "lr": 0.0002998891353088756, "optimizer_lrs": [0.01999260902059171, 0.0002998891353088756], "gradient_norm": 0.6448553800582886, "seconds": 0.24388262000002214, "nonpadding_tokens_per_second": 235465.73347454928, "ranks": [{"compute_seconds": 0.24037900100000797, "nonpadding_tokens": 1624, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23744304899997815, "nonpadding_tokens": 13172, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24013807199997927, "nonpadding_tokens": 11835, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2386728939999898, "nonpadding_tokens": 5138, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23691748999999618, "nonpadding_tokens": 3011, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23838331300004256, "nonpadding_tokens": 4254, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2407825529999741, "nonpadding_tokens": 14281, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24158083700001498, "nonpadding_tokens": 4111, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003657409999959782, "prefetch": {"data_wait_seconds": 0.004340740999865034, "consumed_batches": 461, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1011, "tokens": 84305316, "loss_tokens": 58156, "loss": 3.0107926310268933, "lr": 0.0002998889345379634, "optimizer_lrs": [0.019992595635864228, 0.0002998889345379634], "gradient_norm": 0.5404051542282104, "seconds": 0.23982505699996182, "nonpadding_tokens_per_second": 242493.42719851522, "ranks": [{"compute_seconds": 0.2372489010000436, "nonpadding_tokens": 8867, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2352618360000065, "nonpadding_tokens": 4694, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23529057100000728, "nonpadding_tokens": 2504, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23286798300000555, "nonpadding_tokens": 11303, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.236326350000013, "nonpadding_tokens": 11004, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23334144199998264, "nonpadding_tokens": 10615, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2315511140000126, "nonpadding_tokens": 7443, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23670385100001567, "nonpadding_tokens": 1726, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039931299994577785, "prefetch": {"data_wait_seconds": 0.004349603999870055, "consumed_batches": 462, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1012, "tokens": 84369870, "loss_tokens": 64554, "loss": 2.9119492014437527, "lr": 0.0002998887114667377, "optimizer_lrs": [0.01999258076444918, 0.0002998887114667377], "gradient_norm": 0.41885387897491455, "seconds": 0.2346019740000429, "nonpadding_tokens_per_second": 275163.92509121937, "ranks": [{"compute_seconds": 0.23146727899995767, "nonpadding_tokens": 1969, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2308505390000164, "nonpadding_tokens": 9991, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23051784099999395, "nonpadding_tokens": 11012, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22876893500000506, "nonpadding_tokens": 2751, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23190798399997448, "nonpadding_tokens": 5107, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22910480899997765, "nonpadding_tokens": 6592, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2264990450000255, "nonpadding_tokens": 7591, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23224544099997502, "nonpadding_tokens": 19541, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035894399997005166, "prefetch": {"data_wait_seconds": 0.004358220999861828, "consumed_batches": 463, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1013, "tokens": 84458077, "loss_tokens": 88207, "loss": 2.764071658145045, "lr": 0.0002998884062991214, "optimizer_lrs": [0.01999256041994143, 0.0002998884062991214], "gradient_norm": 0.3933497667312622, "seconds": 0.3349036160000196, "nonpadding_tokens_per_second": 263380.255649419, "ranks": [{"compute_seconds": 0.33167187200001536, "nonpadding_tokens": 16716, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3293184390000192, "nonpadding_tokens": 11587, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33005623999997624, "nonpadding_tokens": 17195, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33220468400003256, "nonpadding_tokens": 10315, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3288548829999627, "nonpadding_tokens": 9464, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32990371400001095, "nonpadding_tokens": 10292, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.33258331600001156, "nonpadding_tokens": 9087, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3293475749999857, "nonpadding_tokens": 3551, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035637000001997876, "prefetch": {"data_wait_seconds": 0.0043676919998603125, "consumed_batches": 464, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1014, "tokens": 84517634, "loss_tokens": 59557, "loss": 2.770007881105496, "lr": 0.0002998882000150105, "optimizer_lrs": [0.01999254666766737, 0.0002998882000150105], "gradient_norm": 0.5738675594329834, "seconds": 0.2672303229999784, "nonpadding_tokens_per_second": 222867.67209425112, "ranks": [{"compute_seconds": 0.26222575600002074, "nonpadding_tokens": 3494, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2610619339999971, "nonpadding_tokens": 6215, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26158567400000265, "nonpadding_tokens": 3390, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2640080709999779, "nonpadding_tokens": 13794, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2649381949999565, "nonpadding_tokens": 5417, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26179835000004914, "nonpadding_tokens": 5132, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2645383020000054, "nonpadding_tokens": 10381, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2610230010000123, "nonpadding_tokens": 11734, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003344799999922543, "prefetch": {"data_wait_seconds": 0.00437699899988786, "consumed_batches": 465, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1015, "tokens": 84563820, "loss_tokens": 46186, "loss": 2.9296672016411898, "lr": 0.0002998880399121701, "optimizer_lrs": [0.019992535994144675, 0.0002998880399121701], "gradient_norm": 0.5400064587593079, "seconds": 0.30271229999999605, "nonpadding_tokens_per_second": 152573.91258961265, "ranks": [{"compute_seconds": 0.29967224100005296, "nonpadding_tokens": 3739, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2997548879999954, "nonpadding_tokens": 3612, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2992988799999807, "nonpadding_tokens": 2578, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29796345599999086, "nonpadding_tokens": 9626, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2960641220000184, "nonpadding_tokens": 16726, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29743309799999906, "nonpadding_tokens": 2368, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2998441459999981, "nonpadding_tokens": 2126, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.300405158999979, "nonpadding_tokens": 5411, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035421699999460543, "prefetch": {"data_wait_seconds": 0.004385163999884867, "consumed_batches": 466, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1016, "tokens": 84612023, "loss_tokens": 48203, "loss": 2.865102081301994, "lr": 0.0002998878726953217, "optimizer_lrs": [0.019992524846354782, 0.0002998878726953217], "gradient_norm": 0.48607146739959717, "seconds": 0.26965246499997875, "nonpadding_tokens_per_second": 178759.72318667214, "ranks": [{"compute_seconds": 0.2655640959999914, "nonpadding_tokens": 2717, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2671033109999712, "nonpadding_tokens": 5571, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26382258699999284, "nonpadding_tokens": 6035, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2663282159999767, "nonpadding_tokens": 4468, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2670506079999768, "nonpadding_tokens": 10967, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2638954650000187, "nonpadding_tokens": 12452, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2628634789999751, "nonpadding_tokens": 4673, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26731678499999134, "nonpadding_tokens": 1320, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039769100004605207, "prefetch": {"data_wait_seconds": 0.004394080999887251, "consumed_batches": 467, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1017, "tokens": 84675049, "loss_tokens": 63026, "loss": 2.6807914590803796, "lr": 0.0002998876538691314, "optimizer_lrs": [0.019992510257942096, 0.0002998876538691314], "gradient_norm": 0.4784894287586212, "seconds": 0.2907024209999918, "nonpadding_tokens_per_second": 216805.8999412385, "ranks": [{"compute_seconds": 0.2869958979999865, "nonpadding_tokens": 2961, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28768240900001274, "nonpadding_tokens": 6650, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2869300109999813, "nonpadding_tokens": 17924, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28717078900001525, "nonpadding_tokens": 5159, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28388266500002146, "nonpadding_tokens": 13161, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28517858200001456, "nonpadding_tokens": 7356, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.28374588599996287, "nonpadding_tokens": 6415, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28834434200001624, "nonpadding_tokens": 3400, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034065199997712625, "prefetch": {"data_wait_seconds": 0.004402902999871117, "consumed_batches": 468, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1018, "tokens": 84727328, "loss_tokens": 52279, "loss": 2.9107280767612234, "lr": 0.0002998874721947186, "optimizer_lrs": [0.019992498146314572, 0.0002998874721947186], "gradient_norm": 0.45616433024406433, "seconds": 0.5385903769999913, "nonpadding_tokens_per_second": 97066.34621138216, "ranks": [{"compute_seconds": 0.5337699510000107, "nonpadding_tokens": 3444, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5362098189999642, "nonpadding_tokens": 10801, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5352205789999971, "nonpadding_tokens": 3074, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.5360167430000047, "nonpadding_tokens": 16285, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.5362514849999798, "nonpadding_tokens": 3416, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5350949250000099, "nonpadding_tokens": 1911, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.5318853720000334, "nonpadding_tokens": 10658, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.5327825279999843, "nonpadding_tokens": 2690, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003840939999690818, "prefetch": {"data_wait_seconds": 0.0044140169998314605, "consumed_batches": 469, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1019, "tokens": 84795512, "loss_tokens": 68184, "loss": 2.8423908559926083, "lr": 0.00029988723502848933, "optimizer_lrs": [0.019992482335232622, 0.00029988723502848933], "gradient_norm": 0.5007338523864746, "seconds": 0.30453083200001174, "nonpadding_tokens_per_second": 223898.51153067275, "ranks": [{"compute_seconds": 0.26019657000000507, "nonpadding_tokens": 12027, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26839300300002833, "nonpadding_tokens": 5018, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27159751799996457, "nonpadding_tokens": 2163, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2662892980000038, "nonpadding_tokens": 3163, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.272609417999945, "nonpadding_tokens": 21783, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2697692429999847, "nonpadding_tokens": 7298, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2722744339999963, "nonpadding_tokens": 5201, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2650714959999618, "nonpadding_tokens": 11531, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.010845990999996502, "prefetch": {"data_wait_seconds": 0.004421740999831059, "consumed_batches": 470, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1020, "tokens": 84857128, "loss_tokens": 61616, "loss": 3.192510873799013, "lr": 0.00029988702049325047, "optimizer_lrs": [0.019992468032883366, 0.00029988702049325047], "gradient_norm": 0.5342108607292175, "seconds": 0.20681942100003425, "nonpadding_tokens_per_second": 297921.731441216, "ranks": [{"compute_seconds": 0.20365462199998774, "nonpadding_tokens": 2355, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20427298799995697, "nonpadding_tokens": 2724, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2030425660000219, "nonpadding_tokens": 20333, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.20189741499996217, "nonpadding_tokens": 3273, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.20123369800001, "nonpadding_tokens": 5156, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2013170619999869, "nonpadding_tokens": 10478, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.20360461500001747, "nonpadding_tokens": 9458, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20447805000003427, "nonpadding_tokens": 7839, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041600700001254154, "prefetch": {"data_wait_seconds": 0.004430522999840036, "consumed_batches": 471, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1021, "tokens": 84920289, "loss_tokens": 63161, "loss": 2.7620389263152894, "lr": 0.0002998868003670878, "optimizer_lrs": [0.019992453357805855, 0.0002998868003670878], "gradient_norm": 0.512356698513031, "seconds": 0.30663213499997255, "nonpadding_tokens_per_second": 205982.9769636038, "ranks": [{"compute_seconds": 0.30311406399999896, "nonpadding_tokens": 6329, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30084835700000667, "nonpadding_tokens": 7171, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.303366063999988, "nonpadding_tokens": 9080, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3014982180000061, "nonpadding_tokens": 16139, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30001101999999946, "nonpadding_tokens": 9113, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.301470514000016, "nonpadding_tokens": 5895, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3041788419999989, "nonpadding_tokens": 5467, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3009293389999925, "nonpadding_tokens": 3967, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003906389999883686, "prefetch": {"data_wait_seconds": 0.0044396879998345185, "consumed_batches": 472, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1022, "tokens": 85000442, "loss_tokens": 80153, "loss": 2.978236349543997, "lr": 0.00029988652071278314, "optimizer_lrs": [0.019992434714185544, 0.00029988652071278314], "gradient_norm": 1.4571629762649536, "seconds": 0.2731966370000123, "nonpadding_tokens_per_second": 293389.4094750383, "ranks": [{"compute_seconds": 0.27007555199998023, "nonpadding_tokens": 4814, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.269850243999997, "nonpadding_tokens": 9153, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26688253399998985, "nonpadding_tokens": 8767, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2702236960000164, "nonpadding_tokens": 4191, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27084599999994907, "nonpadding_tokens": 23311, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2680093650000117, "nonpadding_tokens": 12732, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.270253533000016, "nonpadding_tokens": 10132, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2664512059999993, "nonpadding_tokens": 7053, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004021420000412945, "prefetch": {"data_wait_seconds": 0.004448998999819764, "consumed_batches": 473, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1023, "tokens": 85131625, "loss_tokens": 131183, "loss": 2.6118211296433227, "lr": 0.00029988606227039936, "optimizer_lrs": [0.019992404151359958, 0.00029988606227039936], "gradient_norm": 2.093919038772583, "seconds": 0.3457940609999923, "nonpadding_tokens_per_second": 379367.41776488436, "ranks": [{"compute_seconds": 0.3432356220000088, "nonpadding_tokens": 20348, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34294857099996534, "nonpadding_tokens": 18204, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33927627600002097, "nonpadding_tokens": 10761, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3402209480000238, "nonpadding_tokens": 24193, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34247480100003713, "nonpadding_tokens": 9601, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3411547100000121, "nonpadding_tokens": 17180, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3421760109999923, "nonpadding_tokens": 19828, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3387973090000287, "nonpadding_tokens": 11068, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003963010000234135, "prefetch": {"data_wait_seconds": 0.004459415999818361, "consumed_batches": 474, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1024, "tokens": 85279192, "loss_tokens": 147567, "loss": 2.856957136419389, "lr": 0.00029988554546717296, "optimizer_lrs": [0.01999236969781153, 0.00029988554546717296], "gradient_norm": 1.8764640092849731, "seconds": 0.4035721140000419, "nonpadding_tokens_per_second": 365652.12233664066, "ranks": [{"compute_seconds": 0.398264136000023, "nonpadding_tokens": 18286, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.39731791299999486, "nonpadding_tokens": 12473, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3998933399999487, "nonpadding_tokens": 18344, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.39846346599995286, "nonpadding_tokens": 17069, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4011921089999646, "nonpadding_tokens": 17834, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3994740549999847, "nonpadding_tokens": 18548, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.4007221859999959, "nonpadding_tokens": 25463, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.397123420000014, "nonpadding_tokens": 19550, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004182730000366064, "prefetch": {"data_wait_seconds": 0.004467174999788881, "consumed_batches": 475, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1025, "tokens": 85375865, "loss_tokens": 96673, "loss": 2.688471380323358, "lr": 0.00029988520626914944, "optimizer_lrs": [0.019992347084609965, 0.00029988520626914944], "gradient_norm": 1.0212600231170654, "seconds": 0.35656367500001807, "nonpadding_tokens_per_second": 271124.0846392867, "ranks": [{"compute_seconds": 0.3536343650000049, "nonpadding_tokens": 18993, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35380488799995646, "nonpadding_tokens": 9217, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3507605670000089, "nonpadding_tokens": 19107, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3515345750000165, "nonpadding_tokens": 11158, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35346751699995593, "nonpadding_tokens": 13309, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3510412950000159, "nonpadding_tokens": 5989, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.35354310499997155, "nonpadding_tokens": 8679, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3542352869999945, "nonpadding_tokens": 10221, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034104499997056337, "prefetch": {"data_wait_seconds": 0.004476180999759549, "consumed_batches": 476, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1026, "tokens": 85427593, "loss_tokens": 51728, "loss": 2.850772913122487, "lr": 0.0002998850245642965, "optimizer_lrs": [0.019992334970953103, 0.0002998850245642965], "gradient_norm": 0.5530940890312195, "seconds": 0.33621226500002876, "nonpadding_tokens_per_second": 153855.18431338482, "ranks": [{"compute_seconds": 0.3337353579999558, "nonpadding_tokens": 15573, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3328176709999866, "nonpadding_tokens": 2054, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3299168010000244, "nonpadding_tokens": 5585, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3325547289999804, "nonpadding_tokens": 11135, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3291242509999961, "nonpadding_tokens": 3723, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33206917199999, "nonpadding_tokens": 3062, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3329096219999883, "nonpadding_tokens": 8651, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3337795030000166, "nonpadding_tokens": 1945, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041189999996049664, "prefetch": {"data_wait_seconds": 0.004486671999771374, "consumed_batches": 477, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1027, "tokens": 85481069, "loss_tokens": 53476, "loss": 3.327079846566684, "lr": 0.0002998848365682509, "optimizer_lrs": [0.019992322437883396, 0.0002998848365682509], "gradient_norm": 0.49704721570014954, "seconds": 0.2842797179999934, "nonpadding_tokens_per_second": 188110.50037696055, "ranks": [{"compute_seconds": 0.2799971839999671, "nonpadding_tokens": 2787, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28195051500000545, "nonpadding_tokens": 6077, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28133548900001415, "nonpadding_tokens": 10963, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2818237179999983, "nonpadding_tokens": 3571, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27807664299996304, "nonpadding_tokens": 8226, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2792278780000288, "nonpadding_tokens": 2513, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2818580020000354, "nonpadding_tokens": 10557, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2790632599999867, "nonpadding_tokens": 8782, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003981039999985114, "prefetch": {"data_wait_seconds": 0.004496979999728978, "consumed_batches": 478, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1028, "tokens": 85556725, "loss_tokens": 75656, "loss": 2.8081119227159776, "lr": 0.0002998845703357198, "optimizer_lrs": [0.019992304689047988, 0.0002998845703357198], "gradient_norm": 0.43905773758888245, "seconds": 0.36286881900002754, "nonpadding_tokens_per_second": 208494.07840686984, "ranks": [{"compute_seconds": 0.35915972599997303, "nonpadding_tokens": 11588, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.356361875999994, "nonpadding_tokens": 5792, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3582291040000314, "nonpadding_tokens": 3381, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3595016849999979, "nonpadding_tokens": 4822, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3601520619999974, "nonpadding_tokens": 5140, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3588715070000035, "nonpadding_tokens": 7985, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3598365089999902, "nonpadding_tokens": 25801, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36037114300000894, "nonpadding_tokens": 11147, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005870880000315992, "prefetch": {"data_wait_seconds": 0.004504148999743052, "consumed_batches": 479, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1029, "tokens": 85631999, "loss_tokens": 75274, "loss": 2.833401556314265, "lr": 0.0002998843051425164, "optimizer_lrs": [0.019992287009501095, 0.0002998843051425164], "gradient_norm": 0.5203351378440857, "seconds": 0.26785043500001393, "nonpadding_tokens_per_second": 281029.9710732076, "ranks": [{"compute_seconds": 0.2551070690000188, "nonpadding_tokens": 3992, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.253317141000025, "nonpadding_tokens": 10216, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2532666660000018, "nonpadding_tokens": 4327, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2560690359999853, "nonpadding_tokens": 10834, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25669281400001864, "nonpadding_tokens": 15576, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2538769630000388, "nonpadding_tokens": 14927, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2566575230000012, "nonpadding_tokens": 10557, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2572748000000047, "nonpadding_tokens": 4845, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035291999995479273, "prefetch": {"data_wait_seconds": 0.00451344899973094, "consumed_batches": 480, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1030, "tokens": 85707824, "loss_tokens": 75825, "loss": 3.138214226838114, "lr": 0.0002998840377006177, "optimizer_lrs": [0.01999226918004118, 0.0002998840377006177], "gradient_norm": 0.5026297569274902, "seconds": 0.33299934999996594, "nonpadding_tokens_per_second": 227703.14716832858, "ranks": [{"compute_seconds": 0.3291097979999904, "nonpadding_tokens": 9633, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32736345099999653, "nonpadding_tokens": 13520, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33006882000000815, "nonpadding_tokens": 3667, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3304999450000423, "nonpadding_tokens": 16145, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32722338799999307, "nonpadding_tokens": 9010, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3297664570000052, "nonpadding_tokens": 1143, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3306839640000021, "nonpadding_tokens": 15609, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32764790699997093, "nonpadding_tokens": 7098, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037036300000181654, "prefetch": {"data_wait_seconds": 0.004523543999724211, "consumed_batches": 481, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1031, "tokens": 85763676, "loss_tokens": 55852, "loss": 2.9724447535450835, "lr": 0.0002998838405079787, "optimizer_lrs": [0.019992256033865247, 0.0002998838405079787], "gradient_norm": 0.4775126576423645, "seconds": 0.2624566960000152, "nonpadding_tokens_per_second": 212804.62968259252, "ranks": [{"compute_seconds": 0.23688428499997372, "nonpadding_tokens": 7595, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23678200300003027, "nonpadding_tokens": 2675, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23969517900002302, "nonpadding_tokens": 9395, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24053814799998463, "nonpadding_tokens": 4292, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24084469999996827, "nonpadding_tokens": 13168, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23898990300000378, "nonpadding_tokens": 6090, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.24041890800003785, "nonpadding_tokens": 2659, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23700820900000963, "nonpadding_tokens": 9978, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00840744600003518, "prefetch": {"data_wait_seconds": 0.004532086999702756, "consumed_batches": 482, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1032, "tokens": 85888192, "loss_tokens": 124516, "loss": 2.717564158421408, "lr": 0.0002998834002855185, "optimizer_lrs": [0.019992226685701237, 0.0002998834002855185], "gradient_norm": 0.5691096782684326, "seconds": 0.3591792729999952, "nonpadding_tokens_per_second": 346668.11077375745, "ranks": [{"compute_seconds": 0.3560808090000478, "nonpadding_tokens": 32768, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.356063416999973, "nonpadding_tokens": 12900, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35563364099999717, "nonpadding_tokens": 17414, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.35365368899999794, "nonpadding_tokens": 15868, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35671868299999687, "nonpadding_tokens": 8602, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3537989659999994, "nonpadding_tokens": 23064, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3563610209999979, "nonpadding_tokens": 7351, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35689633599997705, "nonpadding_tokens": 6549, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003537119999919014, "prefetch": {"data_wait_seconds": 0.004540705999730221, "consumed_batches": 483, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1033, "tokens": 85946620, "loss_tokens": 58428, "loss": 2.834123033904977, "lr": 0.00029988319342824903, "optimizer_lrs": [0.019992212895216604, 0.00029988319342824903], "gradient_norm": 0.5216385126113892, "seconds": 0.24372106800001347, "nonpadding_tokens_per_second": 239733.07059362126, "ranks": [{"compute_seconds": 0.2412617530000034, "nonpadding_tokens": 5268, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2396578710000199, "nonpadding_tokens": 4993, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23702734300002248, "nonpadding_tokens": 5263, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24040504300000975, "nonpadding_tokens": 11942, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23573114000004125, "nonpadding_tokens": 4084, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23902330700002494, "nonpadding_tokens": 5022, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2408082219999983, "nonpadding_tokens": 15523, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2412774679999643, "nonpadding_tokens": 6333, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040103800000679257, "prefetch": {"data_wait_seconds": 0.0045642899996920505, "consumed_batches": 484, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1034, "tokens": 86002603, "loss_tokens": 55983, "loss": 2.5635739867459764, "lr": 0.00029988299505529967, "optimizer_lrs": [0.019992199670353313, 0.00029988299505529967], "gradient_norm": 1.7427657842636108, "seconds": 0.3678440009999804, "nonpadding_tokens_per_second": 152192.2332505376, "ranks": [{"compute_seconds": 0.34738898500000914, "nonpadding_tokens": 10248, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35576749500000915, "nonpadding_tokens": 11168, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35245610099997293, "nonpadding_tokens": 10713, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.355377548999968, "nonpadding_tokens": 4211, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35115103199996156, "nonpadding_tokens": 5318, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3544114069999864, "nonpadding_tokens": 6930, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3558131050000384, "nonpadding_tokens": 3290, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35592397700003175, "nonpadding_tokens": 4105, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0019508709999627172, "prefetch": {"data_wait_seconds": 0.004573440999720901, "consumed_batches": 485, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1035, "tokens": 86072086, "loss_tokens": 69483, "loss": 2.8230619360131257, "lr": 0.0002998827486118014, "optimizer_lrs": [0.019992183240786762, 0.0002998827486118014], "gradient_norm": 0.5434261560440063, "seconds": 0.2570647000000008, "nonpadding_tokens_per_second": 270293.8209719179, "ranks": [{"compute_seconds": 0.25429449400002113, "nonpadding_tokens": 1718, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25091734999995197, "nonpadding_tokens": 4440, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25136955400000716, "nonpadding_tokens": 9613, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2536836360000052, "nonpadding_tokens": 7029, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25471286799995596, "nonpadding_tokens": 16464, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25327166799996803, "nonpadding_tokens": 6912, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2540702109999984, "nonpadding_tokens": 8349, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25101752800003396, "nonpadding_tokens": 14958, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041499700000713347, "prefetch": {"data_wait_seconds": 0.004583541999764407, "consumed_batches": 486, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1036, "tokens": 86136789, "loss_tokens": 64703, "loss": 3.004371657805666, "lr": 0.0002998825188890661, "optimizer_lrs": [0.01999216792593774, 0.0002998825188890661], "gradient_norm": 0.4477386772632599, "seconds": 0.28724387699998033, "nonpadding_tokens_per_second": 225254.58392975398, "ranks": [{"compute_seconds": 0.2839883919999693, "nonpadding_tokens": 9523, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28172758599998815, "nonpadding_tokens": 11143, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2841056769999568, "nonpadding_tokens": 6727, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28487953299998026, "nonpadding_tokens": 3429, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27854550400002154, "nonpadding_tokens": 7116, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2827019929999892, "nonpadding_tokens": 8493, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.28482355099998813, "nonpadding_tokens": 9145, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2788205069999776, "nonpadding_tokens": 9127, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035394500002894347, "prefetch": {"data_wait_seconds": 0.004593366999770865, "consumed_batches": 487, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1037, "tokens": 86184512, "loss_tokens": 47723, "loss": 3.011589341093393, "lr": 0.00029988234930843923, "optimizer_lrs": [0.019992156620562616, 0.00029988234930843923], "gradient_norm": 0.46271374821662903, "seconds": 0.26674036300005355, "nonpadding_tokens_per_second": 178911.80570969838, "ranks": [{"compute_seconds": 0.2638068229999817, "nonpadding_tokens": 6772, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26433991299995796, "nonpadding_tokens": 2993, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26298417099997096, "nonpadding_tokens": 7094, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26356553900001245, "nonpadding_tokens": 4195, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26016671000002134, "nonpadding_tokens": 6951, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26151182499995684, "nonpadding_tokens": 3480, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.26384873699998934, "nonpadding_tokens": 9297, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2644136830000434, "nonpadding_tokens": 6941, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042860399997834975, "prefetch": {"data_wait_seconds": 0.004604522999741789, "consumed_batches": 488, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1038, "tokens": 86245132, "loss_tokens": 60620, "loss": 2.8003765774496867, "lr": 0.00029988213372287616, "optimizer_lrs": [0.019992142248191746, 0.00029988213372287616], "gradient_norm": 0.5135045647621155, "seconds": 0.3726506770000242, "nonpadding_tokens_per_second": 162672.453698497, "ranks": [{"compute_seconds": 0.37012584399997195, "nonpadding_tokens": 7727, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36571258499998294, "nonpadding_tokens": 5151, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36668096700003616, "nonpadding_tokens": 5195, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3674785690000135, "nonpadding_tokens": 6518, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.36960895400000027, "nonpadding_tokens": 11796, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3665699100000097, "nonpadding_tokens": 4618, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3653829890000111, "nonpadding_tokens": 8488, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3664234379999698, "nonpadding_tokens": 11127, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003762499999879765, "prefetch": {"data_wait_seconds": 0.004614282999696684, "consumed_batches": 489, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1039, "tokens": 86306116, "loss_tokens": 60984, "loss": 2.8602000114784207, "lr": 0.000299881916643769, "optimizer_lrs": [0.01999212777625127, 0.000299881916643769], "gradient_norm": 0.5647187829017639, "seconds": 0.2076187679999748, "nonpadding_tokens_per_second": 293730.67082262714, "ranks": [{"compute_seconds": 0.20470275700000684, "nonpadding_tokens": 2880, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2053329979999603, "nonpadding_tokens": 7859, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20208675899999662, "nonpadding_tokens": 4542, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.204506662999961, "nonpadding_tokens": 12110, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.20494979000000058, "nonpadding_tokens": 7336, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20256585099997437, "nonpadding_tokens": 9996, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.20509295900001234, "nonpadding_tokens": 13172, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20174119199998586, "nonpadding_tokens": 3089, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003945419999809019, "prefetch": {"data_wait_seconds": 0.004623321999716268, "consumed_batches": 490, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1040, "tokens": 86350477, "loss_tokens": 44361, "loss": 2.75916539161651, "lr": 0.0002998817586105945, "optimizer_lrs": [0.0199921172407063, 0.0002998817586105945], "gradient_norm": 0.473341703414917, "seconds": 0.20617304600000352, "nonpadding_tokens_per_second": 215163.9162376213, "ranks": [{"compute_seconds": 0.2038327879999997, "nonpadding_tokens": 3831, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19893732999997837, "nonpadding_tokens": 10241, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20170643299996982, "nonpadding_tokens": 2847, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.19939625199998545, "nonpadding_tokens": 4044, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2030817340000226, "nonpadding_tokens": 4922, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20011030000000574, "nonpadding_tokens": 4834, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.20244151800000054, "nonpadding_tokens": 3991, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.19798516200000904, "nonpadding_tokens": 9651, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036664300000666117, "prefetch": {"data_wait_seconds": 0.0046339329997096, "consumed_batches": 491, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1041, "tokens": 86407772, "loss_tokens": 57295, "loss": 3.093640915437647, "lr": 0.0002998815543445671, "optimizer_lrs": [0.01999210362297114, 0.0002998815543445671], "gradient_norm": 0.44178256392478943, "seconds": 0.39402019799996424, "nonpadding_tokens_per_second": 145411.3273655205, "ranks": [{"compute_seconds": 0.3893911680000315, "nonpadding_tokens": 11567, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.39039106199999196, "nonpadding_tokens": 6474, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38771196499999405, "nonpadding_tokens": 1991, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.39018704099999013, "nonpadding_tokens": 5215, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3907304680000152, "nonpadding_tokens": 2932, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38930870099994763, "nonpadding_tokens": 2370, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.39038238500000944, "nonpadding_tokens": 17382, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.39141768600001114, "nonpadding_tokens": 9364, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004057280000324681, "prefetch": {"data_wait_seconds": 0.004643702999729271, "consumed_batches": 492, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1042, "tokens": 86519315, "loss_tokens": 111543, "loss": 2.9302813713097193, "lr": 0.00029988115617011747, "optimizer_lrs": [0.019992077078007833, 0.00029988115617011747], "gradient_norm": 2.923159599304199, "seconds": 0.29865836300001547, "nonpadding_tokens_per_second": 373480.2497393794, "ranks": [{"compute_seconds": 0.2954562300000134, "nonpadding_tokens": 11462, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29165274400003227, "nonpadding_tokens": 10859, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29470190600000024, "nonpadding_tokens": 10068, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29297218599998587, "nonpadding_tokens": 10848, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29165107000000035, "nonpadding_tokens": 25504, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2939308159999996, "nonpadding_tokens": 20736, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.29135449800003244, "nonpadding_tokens": 10690, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2959509350000076, "nonpadding_tokens": 11376, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003453100000001541, "prefetch": {"data_wait_seconds": 0.004651999999694567, "consumed_batches": 493, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1043, "tokens": 86601868, "loss_tokens": 82553, "loss": 2.6270672325657456, "lr": 0.00029988086105107244, "optimizer_lrs": [0.01999205740340483, 0.00029988086105107244], "gradient_norm": 2.115180492401123, "seconds": 0.3436704799999575, "nonpadding_tokens_per_second": 240209.74975799554, "ranks": [{"compute_seconds": 0.34103925399995205, "nonpadding_tokens": 20273, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33697655999998233, "nonpadding_tokens": 12551, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3398375350000151, "nonpadding_tokens": 16930, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33854833300000564, "nonpadding_tokens": 6633, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3368695240000079, "nonpadding_tokens": 8693, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33810743899999807, "nonpadding_tokens": 7278, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3408201440000198, "nonpadding_tokens": 5036, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34130866999998943, "nonpadding_tokens": 5159, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003838229999928444, "prefetch": {"data_wait_seconds": 0.0046623919997301755, "consumed_batches": 494, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1044, "tokens": 86641968, "loss_tokens": 40100, "loss": 3.035260481608479, "lr": 0.00029988071756542134, "optimizer_lrs": [0.019992047837694756, 0.00029988071756542134], "gradient_norm": 0.5207405090332031, "seconds": 0.2178280659999814, "nonpadding_tokens_per_second": 184090.14382932376, "ranks": [{"compute_seconds": 0.21531844199995476, "nonpadding_tokens": 6353, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21496370099998785, "nonpadding_tokens": 2434, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21224360000002207, "nonpadding_tokens": 8564, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21256546099999696, "nonpadding_tokens": 5793, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21514814500000057, "nonpadding_tokens": 3176, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2120560579999733, "nonpadding_tokens": 3351, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.21119161499996153, "nonpadding_tokens": 6116, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2155171120000432, "nonpadding_tokens": 4313, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003678549999790448, "prefetch": {"data_wait_seconds": 0.004671300999760319, "consumed_batches": 495, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1045, "tokens": 86734025, "loss_tokens": 92057, "loss": 2.798844737499593, "lr": 0.00029988038784093323, "optimizer_lrs": [0.019992025856062218, 0.00029988038784093323], "gradient_norm": 1.0484681129455566, "seconds": 0.2738431000000219, "nonpadding_tokens_per_second": 336166.95107524213, "ranks": [{"compute_seconds": 0.2715356159999942, "nonpadding_tokens": 4887, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26781938600004196, "nonpadding_tokens": 7009, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26880440100001124, "nonpadding_tokens": 17185, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2665197989999797, "nonpadding_tokens": 11935, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2699284759999614, "nonpadding_tokens": 8694, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26726479499996003, "nonpadding_tokens": 12132, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2694731110000248, "nonpadding_tokens": 4598, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27082743600004733, "nonpadding_tokens": 25617, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040806800001291776, "prefetch": {"data_wait_seconds": 0.004679440999794338, "consumed_batches": 496, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1046, "tokens": 86812203, "loss_tokens": 78178, "loss": 2.993170633042544, "lr": 0.00029988010747026704, "optimizer_lrs": [0.01999200716468447, 0.00029988010747026704], "gradient_norm": 0.4570155441761017, "seconds": 0.3493497069999876, "nonpadding_tokens_per_second": 223781.49582934324, "ranks": [{"compute_seconds": 0.3462830019999501, "nonpadding_tokens": 18351, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34704009699999006, "nonpadding_tokens": 9724, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3459173109999938, "nonpadding_tokens": 24568, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3446655380000152, "nonpadding_tokens": 3150, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34697886700001845, "nonpadding_tokens": 8971, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3443336610000074, "nonpadding_tokens": 2666, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3465870750000022, "nonpadding_tokens": 6273, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34392481100002215, "nonpadding_tokens": 4475, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004112190000000737, "prefetch": {"data_wait_seconds": 0.004689261999786254, "consumed_batches": 497, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1047, "tokens": 86876390, "loss_tokens": 64187, "loss": 2.950916122034057, "lr": 0.00029987987703043587, "optimizer_lrs": [0.01999199180202906, 0.00029987987703043587], "gradient_norm": 0.4711734354496002, "seconds": 0.2735413710000216, "nonpadding_tokens_per_second": 234651.8911027719, "ranks": [{"compute_seconds": 0.2710706369999798, "nonpadding_tokens": 3413, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2671032409999725, "nonpadding_tokens": 13888, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26948444800001425, "nonpadding_tokens": 2371, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2680243049999831, "nonpadding_tokens": 5102, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.270283432000042, "nonpadding_tokens": 3855, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26799307999999655, "nonpadding_tokens": 13938, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.26661433000003854, "nonpadding_tokens": 12789, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26706165000001647, "nonpadding_tokens": 8831, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033937699998887183, "prefetch": {"data_wait_seconds": 0.004697715999739671, "consumed_batches": 498, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1048, "tokens": 86932760, "loss_tokens": 56370, "loss": 3.3848218245520667, "lr": 0.000299879674472281, "optimizer_lrs": [0.01999197829815207, 0.000299879674472281], "gradient_norm": 0.507376492023468, "seconds": 0.3815984999999955, "nonpadding_tokens_per_second": 147720.70644931955, "ranks": [{"compute_seconds": 0.37615205000003016, "nonpadding_tokens": 12220, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3748058030000152, "nonpadding_tokens": 10664, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37757458899994845, "nonpadding_tokens": 2541, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.37524038699996254, "nonpadding_tokens": 2435, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3785729599999854, "nonpadding_tokens": 6006, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3764128790000427, "nonpadding_tokens": 6173, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.37849322900001425, "nonpadding_tokens": 2577, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3791151010000249, "nonpadding_tokens": 13754, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040557500000204527, "prefetch": {"data_wait_seconds": 0.0047071749997513734, "consumed_batches": 499, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1049, "tokens": 86987482, "loss_tokens": 54722, "loss": 2.912213095281605, "lr": 0.00029987947767284086, "optimizer_lrs": [0.01999196517818939, 0.00029987947767284086], "gradient_norm": 0.44446852803230286, "seconds": 0.36298936299999696, "nonpadding_tokens_per_second": 150753.72883585148, "ranks": [{"compute_seconds": 0.3577853219999838, "nonpadding_tokens": 6654, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3567165789999649, "nonpadding_tokens": 16115, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35546169200000577, "nonpadding_tokens": 2100, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3558019809999564, "nonpadding_tokens": 6610, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3604262910000102, "nonpadding_tokens": 4446, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35745586499996307, "nonpadding_tokens": 4914, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.35973579899996366, "nonpadding_tokens": 8748, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.360634986999969, "nonpadding_tokens": 5135, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003643789999614455, "prefetch": {"data_wait_seconds": 0.004715708999768253, "consumed_batches": 500, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1050, "tokens": 87048537, "loss_tokens": 61055, "loss": 2.889513041519941, "lr": 0.00029987925790800454, "optimizer_lrs": [0.019991950527200306, 0.00029987925790800454], "gradient_norm": 0.4408319592475891, "seconds": 0.32419389300002877, "nonpadding_tokens_per_second": 188328.65553082636, "ranks": [{"compute_seconds": 0.2781388790000392, "nonpadding_tokens": 11145, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28541484800001626, "nonpadding_tokens": 6835, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2880241209999781, "nonpadding_tokens": 3652, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28663525700000037, "nonpadding_tokens": 15299, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2854250859999752, "nonpadding_tokens": 2761, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2877089130000172, "nonpadding_tokens": 6967, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2886375890000181, "nonpadding_tokens": 3321, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2860015859999976, "nonpadding_tokens": 11075, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.02118220299996665, "prefetch": {"data_wait_seconds": 0.0047245139997613705, "consumed_batches": 501, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1051, "tokens": 87111995, "loss_tokens": 63458, "loss": 3.1150647869456964, "lr": 0.00029987902928161426, "optimizer_lrs": [0.01999193528544095, 0.00029987902928161426], "gradient_norm": 1.7738829851150513, "seconds": 0.3428433550000136, "nonpadding_tokens_per_second": 185093.27678232957, "ranks": [{"compute_seconds": 0.3377784069999734, "nonpadding_tokens": 23304, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3403031569999939, "nonpadding_tokens": 8070, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3391185830000154, "nonpadding_tokens": 5470, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3396444460000225, "nonpadding_tokens": 4803, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3362384170000041, "nonpadding_tokens": 4468, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33886247999998886, "nonpadding_tokens": 6199, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3398952599999916, "nonpadding_tokens": 2847, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.340496040000005, "nonpadding_tokens": 8297, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040669899999556947, "prefetch": {"data_wait_seconds": 0.004735217999737529, "consumed_batches": 502, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1052, "tokens": 87194233, "loss_tokens": 82238, "loss": 2.959256973661811, "lr": 0.0002998787326730971, "optimizer_lrs": [0.01999191551153981, 0.0002998787326730971], "gradient_norm": 0.44918450713157654, "seconds": 0.32733447399999704, "nonpadding_tokens_per_second": 251235.377059615, "ranks": [{"compute_seconds": 0.3243027789999928, "nonpadding_tokens": 15629, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32403048800000533, "nonpadding_tokens": 3316, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32154084899997315, "nonpadding_tokens": 6193, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32185938899999655, "nonpadding_tokens": 17347, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32027317400002175, "nonpadding_tokens": 9896, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32174725499999113, "nonpadding_tokens": 16536, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3206205100000261, "nonpadding_tokens": 6666, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32495882899996786, "nonpadding_tokens": 6655, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004166319999967527, "prefetch": {"data_wait_seconds": 0.004743972999747825, "consumed_batches": 503, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1053, "tokens": 87320739, "loss_tokens": 126506, "loss": 3.2119615373974355, "lr": 0.0002998782756940676, "optimizer_lrs": [0.019991885046271175, 0.0002998782756940676], "gradient_norm": 0.836193323135376, "seconds": 0.2746458900000448, "nonpadding_tokens_per_second": 460614.93947708217, "ranks": [{"compute_seconds": 0.2712491850000447, "nonpadding_tokens": 6469, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27201531999997997, "nonpadding_tokens": 2697, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2690623900000446, "nonpadding_tokens": 9068, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2695701540000073, "nonpadding_tokens": 11770, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27210691700003053, "nonpadding_tokens": 21149, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26937872000002017, "nonpadding_tokens": 31381, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2678407210000273, "nonpadding_tokens": 26851, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2723576320000234, "nonpadding_tokens": 17121, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034902599998076766, "prefetch": {"data_wait_seconds": 0.004752946999715846, "consumed_batches": 504, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1054, "tokens": 87397052, "loss_tokens": 76313, "loss": 2.8073569133044174, "lr": 0.00029987799961237556, "optimizer_lrs": [0.01999186664082504, 0.00029987799961237556], "gradient_norm": 0.5458096861839294, "seconds": 0.31360705299999836, "nonpadding_tokens_per_second": 243339.55269813526, "ranks": [{"compute_seconds": 0.3110928129999593, "nonpadding_tokens": 11165, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3095103740000127, "nonpadding_tokens": 7172, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3096479499999987, "nonpadding_tokens": 21700, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31021033800004716, "nonpadding_tokens": 10982, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3056150920000391, "nonpadding_tokens": 10611, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3081050519999735, "nonpadding_tokens": 5601, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.30553407799999377, "nonpadding_tokens": 4178, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.311150844999986, "nonpadding_tokens": 4904, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003440790000013294, "prefetch": {"data_wait_seconds": 0.0047620479997050325, "consumed_batches": 505, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1055, "tokens": 87465918, "loss_tokens": 68866, "loss": 2.982064807016525, "lr": 0.0002998777502037757, "optimizer_lrs": [0.019991850013585048, 0.0002998777502037757], "gradient_norm": 0.5246762037277222, "seconds": 0.3517293749999908, "nonpadding_tokens_per_second": 195792.5749022293, "ranks": [{"compute_seconds": 0.3494442990000266, "nonpadding_tokens": 12484, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.346694001000003, "nonpadding_tokens": 1749, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34918283499996505, "nonpadding_tokens": 7361, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34758141099996465, "nonpadding_tokens": 10729, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34596988899994585, "nonpadding_tokens": 9104, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34901908900002354, "nonpadding_tokens": 12416, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3457943010000122, "nonpadding_tokens": 8402, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34683045200000606, "nonpadding_tokens": 6621, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003561760000252434, "prefetch": {"data_wait_seconds": 0.0047693599996847524, "consumed_batches": 506, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1056, "tokens": 87533014, "loss_tokens": 67096, "loss": 2.3301049056575653, "lr": 0.0002998775069606671, "optimizer_lrs": [0.01999183379737781, 0.0002998775069606671], "gradient_norm": 1.046770453453064, "seconds": 0.27305597200000875, "nonpadding_tokens_per_second": 245722.51435686546, "ranks": [{"compute_seconds": 0.27073469399999794, "nonpadding_tokens": 5952, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26670306599999094, "nonpadding_tokens": 6231, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26940316899998606, "nonpadding_tokens": 3845, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26773333900001717, "nonpadding_tokens": 2676, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26611256899997215, "nonpadding_tokens": 9199, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2687343669999791, "nonpadding_tokens": 21542, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2660936419999871, "nonpadding_tokens": 11542, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27059468499999184, "nonpadding_tokens": 6109, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035460500004091955, "prefetch": {"data_wait_seconds": 0.004777713999672528, "consumed_batches": 507, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1057, "tokens": 87579856, "loss_tokens": 46842, "loss": 2.894221531958499, "lr": 0.0002998773370011419, "optimizer_lrs": [0.019991822466742793, 0.0002998773370011419], "gradient_norm": 0.4862745404243469, "seconds": 0.2166556579999792, "nonpadding_tokens_per_second": 216204.83135503664, "ranks": [{"compute_seconds": 0.2128247289999763, "nonpadding_tokens": 1628, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21106451999997944, "nonpadding_tokens": 4968, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21168161400004237, "nonpadding_tokens": 8216, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2141707939999833, "nonpadding_tokens": 5444, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21046303500003205, "nonpadding_tokens": 4675, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2118295960000296, "nonpadding_tokens": 10877, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.21439383599999928, "nonpadding_tokens": 6792, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2113146070000198, "nonpadding_tokens": 4242, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036628100002644715, "prefetch": {"data_wait_seconds": 0.00478697999967892, "consumed_batches": 508, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1058, "tokens": 87653724, "loss_tokens": 73868, "loss": 2.7869224918097144, "lr": 0.0002998770687423211, "optimizer_lrs": [0.01999180458282141, 0.0002998770687423211], "gradient_norm": 0.5516690611839294, "seconds": 0.3028073090000021, "nonpadding_tokens_per_second": 243943.91352026278, "ranks": [{"compute_seconds": 0.30021008500000335, "nonpadding_tokens": 3540, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29684748300002184, "nonpadding_tokens": 11145, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2994530139999938, "nonpadding_tokens": 16622, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29699854200003983, "nonpadding_tokens": 2026, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3004326980000087, "nonpadding_tokens": 15900, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2985715329999721, "nonpadding_tokens": 4238, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2953365189999886, "nonpadding_tokens": 8975, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29632149299999355, "nonpadding_tokens": 11422, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003427159999773721, "prefetch": {"data_wait_seconds": 0.004795810999681294, "consumed_batches": 509, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1059, "tokens": 87734563, "loss_tokens": 80839, "loss": 2.9041758541669243, "lr": 0.0002998767748319915, "optimizer_lrs": [0.019991784988799437, 0.0002998767748319915], "gradient_norm": 0.7014562487602234, "seconds": 0.2524987599999804, "nonpadding_tokens_per_second": 320156.027696953, "ranks": [{"compute_seconds": 0.24888068299998167, "nonpadding_tokens": 5940, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24593949199999088, "nonpadding_tokens": 2947, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24694538300002478, "nonpadding_tokens": 14368, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24711535199998025, "nonpadding_tokens": 6925, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24975062200002185, "nonpadding_tokens": 12577, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24857050299999628, "nonpadding_tokens": 7069, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2456844949999777, "nonpadding_tokens": 15811, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25019173199996203, "nonpadding_tokens": 15202, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035485099999732483, "prefetch": {"data_wait_seconds": 0.004804505999700837, "consumed_batches": 510, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1060, "tokens": 87793578, "loss_tokens": 59015, "loss": 3.045099233245785, "lr": 0.0002998765600467464, "optimizer_lrs": [0.019991770669783096, 0.0002998765600467464], "gradient_norm": 0.49568119645118713, "seconds": 0.36332893299999114, "nonpadding_tokens_per_second": 162428.572678525, "ranks": [{"compute_seconds": 0.3582435889999829, "nonpadding_tokens": 12093, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3602896780000151, "nonpadding_tokens": 3507, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3595621940000342, "nonpadding_tokens": 3025, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3580237570000122, "nonpadding_tokens": 2958, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35633956199995964, "nonpadding_tokens": 13275, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35888249300001007, "nonpadding_tokens": 6855, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3604193879999684, "nonpadding_tokens": 10566, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3609413899999936, "nonpadding_tokens": 6736, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037851500002261673, "prefetch": {"data_wait_seconds": 0.0048136089996688725, "consumed_batches": 511, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1061, "tokens": 87834608, "loss_tokens": 41030, "loss": 2.751942366865708, "lr": 0.0002998764106077901, "optimizer_lrs": [0.01999176070718601, 0.0002998764106077901], "gradient_norm": 0.47112002968788147, "seconds": 0.24974917699995558, "nonpadding_tokens_per_second": 164284.82565132657, "ranks": [{"compute_seconds": 0.2464450960000022, "nonpadding_tokens": 4156, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24377421799999865, "nonpadding_tokens": 3479, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2461476160000302, "nonpadding_tokens": 5704, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24681254400002217, "nonpadding_tokens": 10426, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2474467959999629, "nonpadding_tokens": 5306, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24627497900002027, "nonpadding_tokens": 4436, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.24317934999999125, "nonpadding_tokens": 3232, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24391163400002824, "nonpadding_tokens": 4291, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003491779999649225, "prefetch": {"data_wait_seconds": 0.004822506999701659, "consumed_batches": 512, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1062, "tokens": 87911777, "loss_tokens": 77169, "loss": 2.841947746180461, "lr": 0.0002998761292990212, "optimizer_lrs": [0.019991741953268083, 0.0002998761292990212], "gradient_norm": 0.5540066361427307, "seconds": 0.2607789099999991, "nonpadding_tokens_per_second": 295917.3347261873, "ranks": [{"compute_seconds": 0.2557672529999877, "nonpadding_tokens": 7787, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2542192699999646, "nonpadding_tokens": 9498, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2569897390000051, "nonpadding_tokens": 2975, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2572590540000306, "nonpadding_tokens": 3068, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25360232499997437, "nonpadding_tokens": 12572, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.255338461000008, "nonpadding_tokens": 16994, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2575040180000201, "nonpadding_tokens": 17300, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25834782999999106, "nonpadding_tokens": 6975, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000346251999985725, "prefetch": {"data_wait_seconds": 0.004831400999719904, "consumed_batches": 513, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1063, "tokens": 87956717, "loss_tokens": 44940, "loss": 2.7605442673564755, "lr": 0.0002998759653292785, "optimizer_lrs": [0.0199917310219519, 0.0002998759653292785], "gradient_norm": 0.7579660415649414, "seconds": 0.3812037960000225, "nonpadding_tokens_per_second": 117889.69698506713, "ranks": [{"compute_seconds": 0.3779301590000159, "nonpadding_tokens": 12318, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3786821150000037, "nonpadding_tokens": 7792, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3755943389999743, "nonpadding_tokens": 3960, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3766005689999474, "nonpadding_tokens": 1868, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3746572890000266, "nonpadding_tokens": 6239, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37734262100002525, "nonpadding_tokens": 2567, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.377715903999956, "nonpadding_tokens": 6020, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37574219599997605, "nonpadding_tokens": 4176, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035106000001405846, "prefetch": {"data_wait_seconds": 0.004841085999728989, "consumed_batches": 514, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1064, "tokens": 88057321, "loss_tokens": 100604, "loss": 2.6298854916305516, "lr": 0.0002998755978688446, "optimizer_lrs": [0.01999170652458964, 0.0002998755978688446], "gradient_norm": 0.7600215077400208, "seconds": 0.3844236039999487, "nonpadding_tokens_per_second": 261700.8918110383, "ranks": [{"compute_seconds": 0.38081102500001407, "nonpadding_tokens": 16084, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3806382509999935, "nonpadding_tokens": 5273, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37946875600005114, "nonpadding_tokens": 6735, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3781192330000067, "nonpadding_tokens": 13883, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3807352630000196, "nonpadding_tokens": 1957, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37781557599998905, "nonpadding_tokens": 25774, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3800706580000224, "nonpadding_tokens": 15221, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.38185919699998294, "nonpadding_tokens": 15677, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003548459999933584, "prefetch": {"data_wait_seconds": 0.004849975999732692, "consumed_batches": 515, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1065, "tokens": 88131060, "loss_tokens": 73739, "loss": 2.997135589715076, "lr": 0.0002998753281889563, "optimizer_lrs": [0.01999168854593042, 0.0002998753281889563], "gradient_norm": 0.548337459564209, "seconds": 0.26854034299998375, "nonpadding_tokens_per_second": 274591.88878746785, "ranks": [{"compute_seconds": 0.2661562319999575, "nonpadding_tokens": 6321, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2661840750000124, "nonpadding_tokens": 3568, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2653828339999791, "nonpadding_tokens": 23167, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26615370700000085, "nonpadding_tokens": 6698, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2624704000000406, "nonpadding_tokens": 7391, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26509351400000014, "nonpadding_tokens": 7637, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.26619535700001506, "nonpadding_tokens": 16458, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2628740370000173, "nonpadding_tokens": 2499, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004017200000134835, "prefetch": {"data_wait_seconds": 0.004858752999737703, "consumed_batches": 516, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1066, "tokens": 88191508, "loss_tokens": 60448, "loss": 2.884104933165696, "lr": 0.00029987510689951594, "optimizer_lrs": [0.019991673793301065, 0.00029987510689951594], "gradient_norm": 0.6920504570007324, "seconds": 0.3107065289999582, "nonpadding_tokens_per_second": 194550.14413298064, "ranks": [{"compute_seconds": 0.24770500599998968, "nonpadding_tokens": 14870, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2655985209999585, "nonpadding_tokens": 1497, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2630033249999997, "nonpadding_tokens": 5000, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26547065599999087, "nonpadding_tokens": 5038, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2620967600000199, "nonpadding_tokens": 4080, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26341679600000134, "nonpadding_tokens": 16750, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.26583349800000633, "nonpadding_tokens": 9659, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26642550899998696, "nonpadding_tokens": 3554, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.020107596999991983, "prefetch": {"data_wait_seconds": 0.004868669999737563, "consumed_batches": 517, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1067, "tokens": 88285102, "loss_tokens": 93594, "loss": 2.7012251720195737, "lr": 0.0002998747638815022, "optimizer_lrs": [0.019991650925433483, 0.0002998747638815022], "gradient_norm": 0.5919478535652161, "seconds": 0.27773376400000416, "nonpadding_tokens_per_second": 336991.79621530854, "ranks": [{"compute_seconds": 0.27305915199997344, "nonpadding_tokens": 2525, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27485763799995766, "nonpadding_tokens": 21801, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2720393490000106, "nonpadding_tokens": 6929, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27323436000000356, "nonpadding_tokens": 2824, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2711803669999995, "nonpadding_tokens": 12228, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2742513520000216, "nonpadding_tokens": 17783, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.27130738799996834, "nonpadding_tokens": 17996, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27538412900003095, "nonpadding_tokens": 11508, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004051649999610163, "prefetch": {"data_wait_seconds": 0.0048776839997373145, "consumed_batches": 518, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1068, "tokens": 88366354, "loss_tokens": 81252, "loss": 2.9272654442352186, "lr": 0.00029987446571512255, "optimizer_lrs": [0.01999163104767484, 0.00029987446571512255], "gradient_norm": 0.4245944023132324, "seconds": 0.2779661750000173, "nonpadding_tokens_per_second": 292308.9473026527, "ranks": [{"compute_seconds": 0.27557034599999497, "nonpadding_tokens": 6436, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2716304040000068, "nonpadding_tokens": 2523, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2726448459999915, "nonpadding_tokens": 12570, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27477916300000516, "nonpadding_tokens": 11387, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2756348990000106, "nonpadding_tokens": 3967, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2725753919999647, "nonpadding_tokens": 8960, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2711118889999966, "nonpadding_tokens": 17624, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2719739929999605, "nonpadding_tokens": 17785, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003829789999940658, "prefetch": {"data_wait_seconds": 0.004886596999725157, "consumed_batches": 519, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1069, "tokens": 88453434, "loss_tokens": 87080, "loss": 2.8781346893661, "lr": 0.00029987414576863186, "optimizer_lrs": [0.019991609717908792, 0.00029987414576863186], "gradient_norm": 0.41394588351249695, "seconds": 0.40533705199999304, "nonpadding_tokens_per_second": 214833.55535926062, "ranks": [{"compute_seconds": 0.39913417400003937, "nonpadding_tokens": 18047, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4016258139999991, "nonpadding_tokens": 8041, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4013317320000169, "nonpadding_tokens": 18204, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3987283700000148, "nonpadding_tokens": 11266, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.401425598000003, "nonpadding_tokens": 5189, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3985436170000298, "nonpadding_tokens": 8717, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.39876880399998527, "nonpadding_tokens": 5377, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4029977519999761, "nonpadding_tokens": 12239, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003474199999686789, "prefetch": {"data_wait_seconds": 0.0048952699996931415, "consumed_batches": 520, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1070, "tokens": 88502368, "loss_tokens": 48934, "loss": 2.7691121587239955, "lr": 0.0002998739657983063, "optimizer_lrs": [0.01999159771988709, 0.0002998739657983063], "gradient_norm": 0.845410943031311, "seconds": 0.24772451699999465, "nonpadding_tokens_per_second": 197533.94049407332, "ranks": [{"compute_seconds": 0.24542263699999012, "nonpadding_tokens": 5047, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24546298199999228, "nonpadding_tokens": 3687, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24411856800003306, "nonpadding_tokens": 3142, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24527652900002295, "nonpadding_tokens": 3380, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24141967700001032, "nonpadding_tokens": 21746, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24253118699999732, "nonpadding_tokens": 4249, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.24498692999998184, "nonpadding_tokens": 4384, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24222750100000212, "nonpadding_tokens": 3299, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003750230000036936, "prefetch": {"data_wait_seconds": 0.0049044939996889525, "consumed_batches": 521, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1071, "tokens": 88570153, "loss_tokens": 67785, "loss": 3.040169976764771, "lr": 0.00029987371628511004, "optimizer_lrs": [0.019991581085674004, 0.00029987371628511004], "gradient_norm": 0.5337703824043274, "seconds": 0.2945090870000513, "nonpadding_tokens_per_second": 230162.67745921263, "ranks": [{"compute_seconds": 0.29203609199998937, "nonpadding_tokens": 9300, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28675883200003227, "nonpadding_tokens": 3787, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2892885539999952, "nonpadding_tokens": 8809, "peak_allocated_bytes": 14997529088, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2897491969999919, "nonpadding_tokens": 11138, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2876907939999569, "nonpadding_tokens": 21097, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2909159870000053, "nonpadding_tokens": 1741, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2901047619999986, "nonpadding_tokens": 8482, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2910262710000211, "nonpadding_tokens": 3431, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004216639999867766, "prefetch": {"data_wait_seconds": 0.004913963999698012, "consumed_batches": 522, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1072, "tokens": 88623545, "loss_tokens": 53392, "loss": 2.7579286807948757, "lr": 0.0002998735195781932, "optimizer_lrs": [0.01999156797187955, 0.0002998735195781932], "gradient_norm": 0.46819567680358887, "seconds": 0.2668503590000455, "nonpadding_tokens_per_second": 200082.1741446145, "ranks": [{"compute_seconds": 0.26374549000001934, "nonpadding_tokens": 3414, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26453976199996987, "nonpadding_tokens": 9437, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2638073979999831, "nonpadding_tokens": 9921, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26399924699995836, "nonpadding_tokens": 4930, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26037122999997564, "nonpadding_tokens": 12490, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26201478699999825, "nonpadding_tokens": 5729, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2643593340000052, "nonpadding_tokens": 1646, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2611379840000154, "nonpadding_tokens": 5825, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003923719999647801, "prefetch": {"data_wait_seconds": 0.004922797999725503, "consumed_batches": 523, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1073, "tokens": 88699067, "loss_tokens": 75522, "loss": 2.9540357445512564, "lr": 0.00029987324107860156, "optimizer_lrs": [0.019991549405240106, 0.00029987324107860156], "gradient_norm": 0.844365656375885, "seconds": 0.2529012360000138, "nonpadding_tokens_per_second": 298622.50258039817, "ranks": [{"compute_seconds": 0.24948552999995854, "nonpadding_tokens": 9108, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2463118379999969, "nonpadding_tokens": 6693, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24919758000004322, "nonpadding_tokens": 3409, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24678884999997308, "nonpadding_tokens": 15574, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24520647399998552, "nonpadding_tokens": 21885, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.247935158999951, "nonpadding_tokens": 2149, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2500021399999923, "nonpadding_tokens": 6323, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2505719010000007, "nonpadding_tokens": 10381, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033235000000786385, "prefetch": {"data_wait_seconds": 0.004930973999719868, "consumed_batches": 524, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1074, "tokens": 88761563, "loss_tokens": 62496, "loss": 2.606343806003584, "lr": 0.0002998730103830059, "optimizer_lrs": [0.01999153402553373, 0.0002998730103830059], "gradient_norm": 0.5616887807846069, "seconds": 0.3911761830000273, "nonpadding_tokens_per_second": 159764.3279831166, "ranks": [{"compute_seconds": 0.3884084409999673, "nonpadding_tokens": 9452, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.38883839999999736, "nonpadding_tokens": 9947, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3856867459999762, "nonpadding_tokens": 3156, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.38652472399996896, "nonpadding_tokens": 4650, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.38509682699998393, "nonpadding_tokens": 11688, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3860271740000485, "nonpadding_tokens": 6864, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.38487424099997725, "nonpadding_tokens": 13153, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.38546048400002064, "nonpadding_tokens": 3586, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003534299999614632, "prefetch": {"data_wait_seconds": 0.0049396689997394105, "consumed_batches": 525, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1075, "tokens": 88823419, "loss_tokens": 61856, "loss": 3.0243286827470253, "lr": 0.00029987278184345545, "optimizer_lrs": [0.0199915187895637, 0.00029987278184345545], "gradient_norm": 0.5396960377693176, "seconds": 0.29878800499994895, "nonpadding_tokens_per_second": 207023.03628290087, "ranks": [{"compute_seconds": 0.29488475800002334, "nonpadding_tokens": 9135, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29622926600001165, "nonpadding_tokens": 1172, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29430488299999524, "nonpadding_tokens": 2172, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29448606400001154, "nonpadding_tokens": 1887, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29245946099996445, "nonpadding_tokens": 8758, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2924125600000025, "nonpadding_tokens": 19235, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.2908535579999807, "nonpadding_tokens": 11800, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2954794940000056, "nonpadding_tokens": 7697, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039559199996119787, "prefetch": {"data_wait_seconds": 0.004947988999745689, "consumed_batches": 526, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1076, "tokens": 88867103, "loss_tokens": 43684, "loss": 2.9721812477108323, "lr": 0.0002998726203203184, "optimizer_lrs": [0.019991508021354562, 0.0002998726203203184], "gradient_norm": 0.5956493020057678, "seconds": 0.346164817999977, "nonpadding_tokens_per_second": 126194.22231407382, "ranks": [{"compute_seconds": 0.33987461199996005, "nonpadding_tokens": 10690, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3425730779999867, "nonpadding_tokens": 4391, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34283218999996734, "nonpadding_tokens": 4296, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3430149410000354, "nonpadding_tokens": 1826, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3437042760000395, "nonpadding_tokens": 1995, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34118361199995206, "nonpadding_tokens": 4375, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.33819625399996767, "nonpadding_tokens": 7043, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.338867123, "nonpadding_tokens": 9068, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003782539999974688, "prefetch": {"data_wait_seconds": 0.004957119999701831, "consumed_batches": 527, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1077, "tokens": 88920864, "loss_tokens": 53761, "loss": 2.762580274269452, "lr": 0.00029987242139652457, "optimizer_lrs": [0.019991494759768306, 0.00029987242139652457], "gradient_norm": 0.4830690324306488, "seconds": 0.24426410899997109, "nonpadding_tokens_per_second": 220093.7346878348, "ranks": [{"compute_seconds": 0.23954465399998526, "nonpadding_tokens": 8176, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2407629769999744, "nonpadding_tokens": 15017, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2396198019999929, "nonpadding_tokens": 4473, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23833125800001653, "nonpadding_tokens": 11012, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23719575399996984, "nonpadding_tokens": 7713, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23865785199996026, "nonpadding_tokens": 1469, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.23694825299998, "nonpadding_tokens": 2631, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24189033099997914, "nonpadding_tokens": 3270, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038587099999176644, "prefetch": {"data_wait_seconds": 0.004967042999751925, "consumed_batches": 528, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1078, "tokens": 89020023, "loss_tokens": 99159, "loss": 2.856821178612128, "lr": 0.00029987205408635596, "optimizer_lrs": [0.019991470272423734, 0.00029987205408635596], "gradient_norm": 1.4297258853912354, "seconds": 0.3123182929999757, "nonpadding_tokens_per_second": 317493.41048046685, "ranks": [{"compute_seconds": 0.29513638499997796, "nonpadding_tokens": 14860, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3090218430000391, "nonpadding_tokens": 19612, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30831211100002065, "nonpadding_tokens": 11067, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30856863500002873, "nonpadding_tokens": 8366, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30479871400001457, "nonpadding_tokens": 4478, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30668826800001625, "nonpadding_tokens": 4782, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.3098316380000483, "nonpadding_tokens": 12433, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.309965845000022, "nonpadding_tokens": 23561, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033709400003090195, "prefetch": {"data_wait_seconds": 0.00497523799977273, "consumed_batches": 529, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1079, "tokens": 89061542, "loss_tokens": 41519, "loss": 3.2053246706327223, "lr": 0.0002998719001326652, "optimizer_lrs": [0.019991460008844348, 0.0002998719001326652], "gradient_norm": 0.5224775075912476, "seconds": 0.361428850999971, "nonpadding_tokens_per_second": 114874.61469976377, "ranks": [{"compute_seconds": 0.35603824099996473, "nonpadding_tokens": 2310, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.358373959000005, "nonpadding_tokens": 3132, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3574494079999795, "nonpadding_tokens": 5884, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3584392610000009, "nonpadding_tokens": 5500, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35868074000001116, "nonpadding_tokens": 3772, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3558152080000241, "nonpadding_tokens": 6676, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.35423229200000605, "nonpadding_tokens": 3118, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3590547709999896, "nonpadding_tokens": 11127, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034637300001350013, "prefetch": {"data_wait_seconds": 0.004983866999808129, "consumed_batches": 530, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1080, "tokens": 89137388, "loss_tokens": 75846, "loss": 2.646988634865385, "lr": 0.00029987161865454117, "optimizer_lrs": [0.01999144124363608, 0.00029987161865454117], "gradient_norm": 0.5480132102966309, "seconds": 0.35073920800005, "nonpadding_tokens_per_second": 216246.14035163468, "ranks": [{"compute_seconds": 0.34551619400002664, "nonpadding_tokens": 15368, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3444238199999745, "nonpadding_tokens": 9054, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34698065399999223, "nonpadding_tokens": 10140, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34730954100001554, "nonpadding_tokens": 11148, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34786989600002016, "nonpadding_tokens": 3326, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3466204599999969, "nonpadding_tokens": 4640, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.34788107899998977, "nonpadding_tokens": 11267, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34833071600002086, "nonpadding_tokens": 10903, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003945859999703316, "prefetch": {"data_wait_seconds": 0.00499191599983817, "consumed_batches": 531, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1081, "tokens": 89177086, "loss_tokens": 39698, "loss": 2.9892325731145144, "lr": 0.00029987147120503837, "optimizer_lrs": [0.019991431413669224, 0.00029987147120503837], "gradient_norm": 0.4667273163795471, "seconds": 0.22009481100002404, "nonpadding_tokens_per_second": 180367.72343531382, "ranks": [{"compute_seconds": 0.21707280999999057, "nonpadding_tokens": 2460, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21772015799996325, "nonpadding_tokens": 2036, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21657572100002653, "nonpadding_tokens": 3843, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21551910400000907, "nonpadding_tokens": 4867, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21383946200000992, "nonpadding_tokens": 8844, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21612729099996386, "nonpadding_tokens": 2133, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19058917376}, {"compute_seconds": 0.21729344599998512, "nonpadding_tokens": 5285, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2177794199999994, "nonpadding_tokens": 10230, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036230300003126104, "prefetch": {"data_wait_seconds": 0.005000473999828614, "consumed_batches": 532, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1082, "tokens": 89238353, "loss_tokens": 61267, "loss": 2.8392007524442198, "lr": 0.0002998712434762141, "optimizer_lrs": [0.019991416231747605, 0.0002998712434762141], "gradient_norm": 0.4397449791431427, "seconds": 0.21643187600000147, "nonpadding_tokens_per_second": 283077.5259740371, "ranks": [{"compute_seconds": 0.17890420599997015, "nonpadding_tokens": 7913, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.18507839599999443, "nonpadding_tokens": 2205, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.18631525099999635, "nonpadding_tokens": 10137, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.18820322900000974, "nonpadding_tokens": 4139, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.18763697800000045, "nonpadding_tokens": 2280, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1846037120000119, "nonpadding_tokens": 8549, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.18685117800004036, "nonpadding_tokens": 15935, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.18094918499997448, "nonpadding_tokens": 10109, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.014570237999976143, "prefetch": {"data_wait_seconds": 0.005009307999856105, "consumed_batches": 533, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1083, "tokens": 89361227, "loss_tokens": 122874, "loss": 2.767922068541758, "lr": 0.0002998707861475764, "optimizer_lrs": [0.01999138574317176, 0.0002998707861475764], "gradient_norm": 0.6236993074417114, "seconds": 0.29036579699999265, "nonpadding_tokens_per_second": 423169.6751804522, "ranks": [{"compute_seconds": 0.2880055339999785, "nonpadding_tokens": 5050, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2862588080000137, "nonpadding_tokens": 10580, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2830984369999783, "nonpadding_tokens": 10803, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2836399100000335, "nonpadding_tokens": 13525, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2819393280000213, "nonpadding_tokens": 17807, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2844520849999981, "nonpadding_tokens": 18449, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.287159097999961, "nonpadding_tokens": 24048, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2877861709999934, "nonpadding_tokens": 22612, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038727999998400264, "prefetch": {"data_wait_seconds": 0.005018835999820226, "consumed_batches": 534, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1084, "tokens": 89440184, "loss_tokens": 78957, "loss": 2.868968631660271, "lr": 0.000299870491847345, "optimizer_lrs": [0.019991366123156336, 0.000299870491847345], "gradient_norm": 0.49490317702293396, "seconds": 0.2770246330000532, "nonpadding_tokens_per_second": 285017.97527869965, "ranks": [{"compute_seconds": 0.2452340670000126, "nonpadding_tokens": 3635, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24696493499999406, "nonpadding_tokens": 16442, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2462604899999974, "nonpadding_tokens": 8760, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24473388299998078, "nonpadding_tokens": 21658, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24327356100002362, "nonpadding_tokens": 6283, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24450373700000227, "nonpadding_tokens": 6456, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24326378200004228, "nonpadding_tokens": 8976, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2479127170000197, "nonpadding_tokens": 6747, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01478046599999061, "prefetch": {"data_wait_seconds": 0.005028496999798904, "consumed_batches": 535, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1085, "tokens": 89500827, "loss_tokens": 60643, "loss": 2.4372413139191664, "lr": 0.0002998702655825759, "optimizer_lrs": [0.019991351038838395, 0.0002998702655825759], "gradient_norm": 0.47675374150276184, "seconds": 0.3426973569999632, "nonpadding_tokens_per_second": 176957.88648876717, "ranks": [{"compute_seconds": 0.33864780300001485, "nonpadding_tokens": 19309, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33698954299995876, "nonpadding_tokens": 2819, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3375658209999983, "nonpadding_tokens": 7097, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33782762800001365, "nonpadding_tokens": 11101, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33629896200000076, "nonpadding_tokens": 2216, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33953606599999375, "nonpadding_tokens": 8166, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.34034092400003146, "nonpadding_tokens": 5983, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33716213900004277, "nonpadding_tokens": 3952, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038896200004501225, "prefetch": {"data_wait_seconds": 0.005037938999819858, "consumed_batches": 536, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1086, "tokens": 89527947, "loss_tokens": 27120, "loss": 3.230795423119469, "lr": 0.000299870164331416, "optimizer_lrs": [0.019991344288761065, 0.000299870164331416], "gradient_norm": 0.5493960976600647, "seconds": 0.27819539799997983, "nonpadding_tokens_per_second": 97485.43719620396, "ranks": [{"compute_seconds": 0.2744175540000242, "nonpadding_tokens": 2120, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2742724839999937, "nonpadding_tokens": 4486, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27164710400001013, "nonpadding_tokens": 2280, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2722874880000177, "nonpadding_tokens": 3461, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27569975699998395, "nonpadding_tokens": 3670, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2727419119999581, "nonpadding_tokens": 3039, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27486517199997706, "nonpadding_tokens": 4494, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27548464599999534, "nonpadding_tokens": 3570, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003448380000463658, "prefetch": {"data_wait_seconds": 0.005045869999847241, "consumed_batches": 537, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1087, "tokens": 89582831, "loss_tokens": 54884, "loss": 2.9113044102106262, "lr": 0.000299869959303971, "optimizer_lrs": [0.019991330620264734, 0.000299869959303971], "gradient_norm": 0.4539964497089386, "seconds": 0.214172743000006, "nonpadding_tokens_per_second": 256260.43366311307, "ranks": [{"compute_seconds": 0.21064263399995298, "nonpadding_tokens": 3669, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2078017429999477, "nonpadding_tokens": 6093, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21006863700000622, "nonpadding_tokens": 10553, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.20817290100001173, "nonpadding_tokens": 8880, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.20646998400002303, "nonpadding_tokens": 9790, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20957777299997815, "nonpadding_tokens": 10006, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.20659302799998613, "nonpadding_tokens": 3521, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2118005159999825, "nonpadding_tokens": 2372, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000386708999997154, "prefetch": {"data_wait_seconds": 0.005054694999856224, "consumed_batches": 538, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1088, "tokens": 89644873, "loss_tokens": 62042, "loss": 3.050092830260146, "lr": 0.0002998697273420585, "optimizer_lrs": [0.019991315156137234, 0.0002998697273420585], "gradient_norm": 0.5006718039512634, "seconds": 0.28031984099999363, "nonpadding_tokens_per_second": 221325.75339182434, "ranks": [{"compute_seconds": 0.27692242300003045, "nonpadding_tokens": 3544, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27802820199997313, "nonpadding_tokens": 3109, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2754115209999668, "nonpadding_tokens": 5341, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27585450099996933, "nonpadding_tokens": 16140, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2740986310000153, "nonpadding_tokens": 16081, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27577660800000103, "nonpadding_tokens": 12036, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2779529679999655, "nonpadding_tokens": 2588, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27471788499997274, "nonpadding_tokens": 3203, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003888370000026953, "prefetch": {"data_wait_seconds": 0.0050635859998493515, "consumed_batches": 539, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1089, "tokens": 89714416, "loss_tokens": 69543, "loss": 2.869736619789195, "lr": 0.0002998694670899183, "optimizer_lrs": [0.019991297805994554, 0.0002998694670899183], "gradient_norm": 0.506266176700592, "seconds": 0.3012973550000311, "nonpadding_tokens_per_second": 230811.85030646162, "ranks": [{"compute_seconds": 0.29727015599996776, "nonpadding_tokens": 5757, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2982671319999781, "nonpadding_tokens": 8798, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29534453200000144, "nonpadding_tokens": 14324, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29849294199999576, "nonpadding_tokens": 17549, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29420023500000525, "nonpadding_tokens": 6282, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2959682079999766, "nonpadding_tokens": 3796, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2945591830000467, "nonpadding_tokens": 9006, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2989974110000162, "nonpadding_tokens": 4031, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000340626000024713, "prefetch": {"data_wait_seconds": 0.005070738999862101, "consumed_batches": 540, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1090, "tokens": 89790141, "loss_tokens": 75725, "loss": 2.8426745625619017, "lr": 0.0002998691834075456, "optimizer_lrs": [0.019991278893836374, 0.0002998691834075456], "gradient_norm": 0.6504219174385071, "seconds": 0.4214725130000261, "nonpadding_tokens_per_second": 179667.7070611134, "ranks": [{"compute_seconds": 0.41702103799997303, "nonpadding_tokens": 8290, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.41837685500001953, "nonpadding_tokens": 3978, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41515995200001043, "nonpadding_tokens": 8518, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.41826317099997823, "nonpadding_tokens": 3225, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4140396399999986, "nonpadding_tokens": 6584, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41593937499999356, "nonpadding_tokens": 15423, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.41867114499996205, "nonpadding_tokens": 12901, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.41902341799999476, "nonpadding_tokens": 16806, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033495299999231065, "prefetch": {"data_wait_seconds": 0.005079634999859195, "consumed_batches": 541, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1091, "tokens": 89828737, "loss_tokens": 38596, "loss": 2.768407631295989, "lr": 0.00029986903870009713, "optimizer_lrs": [0.019991269246673143, 0.00029986903870009713], "gradient_norm": 0.4931185841560364, "seconds": 0.30040236500002493, "nonpadding_tokens_per_second": 128481.01245806369, "ranks": [{"compute_seconds": 0.29652331800002685, "nonpadding_tokens": 2272, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29364432400001306, "nonpadding_tokens": 3726, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29342623399998047, "nonpadding_tokens": 3265, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29400560400000586, "nonpadding_tokens": 6975, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2975668440000163, "nonpadding_tokens": 4238, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2959066780000512, "nonpadding_tokens": 10334, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.29692865999999185, "nonpadding_tokens": 4803, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29788934900000186, "nonpadding_tokens": 2983, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004313679999654596, "prefetch": {"data_wait_seconds": 0.005088781999873504, "consumed_batches": 542, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1092, "tokens": 89904347, "loss_tokens": 75610, "loss": 2.801241155270467, "lr": 0.0002998687549848361, "optimizer_lrs": [0.019991250332322408, 0.0002998687549848361], "gradient_norm": 0.45783331990242004, "seconds": 0.2965606219999586, "nonpadding_tokens_per_second": 254956.3036727464, "ranks": [{"compute_seconds": 0.2941074720000074, "nonpadding_tokens": 15642, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29100066899997046, "nonpadding_tokens": 6706, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29396385299997974, "nonpadding_tokens": 16854, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2942256080000334, "nonpadding_tokens": 4810, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2902842180000107, "nonpadding_tokens": 9840, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2923387139999818, "nonpadding_tokens": 17049, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2898397989999921, "nonpadding_tokens": 2195, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29048324299998285, "nonpadding_tokens": 2514, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004097610000144414, "prefetch": {"data_wait_seconds": 0.005097971999830975, "consumed_batches": 543, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1093, "tokens": 89977086, "loss_tokens": 72739, "loss": 3.0383301083325316, "lr": 0.000299868481752991, "optimizer_lrs": [0.01999123211686607, 0.000299868481752991], "gradient_norm": 0.4796285033226013, "seconds": 0.29870403099999976, "nonpadding_tokens_per_second": 243515.29424120847, "ranks": [{"compute_seconds": 0.29544952899999544, "nonpadding_tokens": 1940, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2923098550000418, "nonpadding_tokens": 17061, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2926826450000135, "nonpadding_tokens": 18931, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29335663099999465, "nonpadding_tokens": 5939, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29600169700000833, "nonpadding_tokens": 4429, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.294643313999984, "nonpadding_tokens": 5154, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.29166510400000334, "nonpadding_tokens": 8938, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2959720240000365, "nonpadding_tokens": 10347, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042582200001106685, "prefetch": {"data_wait_seconds": 0.005109030999804531, "consumed_batches": 544, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1094, "tokens": 90090689, "loss_tokens": 113603, "loss": 2.399136961611929, "lr": 0.00029986805445429216, "optimizer_lrs": [0.019991203630286145, 0.00029986805445429216], "gradient_norm": 0.42988982796669006, "seconds": 0.30128543199998603, "nonpadding_tokens_per_second": 377061.0455536571, "ranks": [{"compute_seconds": 0.2976844429999801, "nonpadding_tokens": 10450, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2949877949999973, "nonpadding_tokens": 5214, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29548975800003063, "nonpadding_tokens": 19582, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29870128900000736, "nonpadding_tokens": 4750, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29869487700000263, "nonpadding_tokens": 13824, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.296015868999973, "nonpadding_tokens": 22251, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2945856520000234, "nonpadding_tokens": 21277, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29897557400005326, "nonpadding_tokens": 16255, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038686199997073345, "prefetch": {"data_wait_seconds": 0.005118299999779197, "consumed_batches": 545, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1095, "tokens": 90195946, "loss_tokens": 105257, "loss": 2.5672787795586043, "lr": 0.00029986765792946535, "optimizer_lrs": [0.01999117719529769, 0.00029986765792946535], "gradient_norm": 0.7449188232421875, "seconds": 0.39458229699999947, "nonpadding_tokens_per_second": 266755.50525268534, "ranks": [{"compute_seconds": 0.3895880590000047, "nonpadding_tokens": 11321, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3922252820000267, "nonpadding_tokens": 18140, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3898242159999654, "nonpadding_tokens": 20039, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3903995009999903, "nonpadding_tokens": 25173, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3883609990000423, "nonpadding_tokens": 17854, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3910962379999887, "nonpadding_tokens": 4193, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3903636430000006, "nonpadding_tokens": 3854, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3891902889999983, "nonpadding_tokens": 4683, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040560799999411756, "prefetch": {"data_wait_seconds": 0.005127404999782925, "consumed_batches": 546, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1096, "tokens": 90238949, "loss_tokens": 43003, "loss": 2.690882213159547, "lr": 0.0002998674957571988, "optimizer_lrs": [0.019991166383813253, 0.0002998674957571988], "gradient_norm": 0.5942780375480652, "seconds": 0.2748052590000043, "nonpadding_tokens_per_second": 156485.3604202652, "ranks": [{"compute_seconds": 0.2707550339999898, "nonpadding_tokens": 5077, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27094759800002066, "nonpadding_tokens": 6302, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2709937759999548, "nonpadding_tokens": 2521, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27161799200001724, "nonpadding_tokens": 3165, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27235604899999544, "nonpadding_tokens": 4880, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2694626730000209, "nonpadding_tokens": 14314, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27172950199997103, "nonpadding_tokens": 3401, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26724974300003623, "nonpadding_tokens": 3343, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038063200003080055, "prefetch": {"data_wait_seconds": 0.005137214999777484, "consumed_batches": 547, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1097, "tokens": 90307927, "loss_tokens": 68978, "loss": 2.8796948846008874, "lr": 0.0002998672354211155, "optimizer_lrs": [0.019991149028074365, 0.0002998672354211155], "gradient_norm": 0.4440740644931793, "seconds": 0.2841607909999766, "nonpadding_tokens_per_second": 242742.8490653578, "ranks": [{"compute_seconds": 0.27926243000001705, "nonpadding_tokens": 11429, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27754124400001956, "nonpadding_tokens": 11676, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27823351900002535, "nonpadding_tokens": 1817, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2787981930000001, "nonpadding_tokens": 7533, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2809067660000437, "nonpadding_tokens": 2790, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27977866300000187, "nonpadding_tokens": 9530, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2771754159999773, "nonpadding_tokens": 12494, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2818156760000079, "nonpadding_tokens": 11709, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004048430000125336, "prefetch": {"data_wait_seconds": 0.005147458999729793, "consumed_batches": 548, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1098, "tokens": 90351718, "loss_tokens": 43791, "loss": 3.0002982918864607, "lr": 0.0002998670700130344, "optimizer_lrs": [0.019991138000868963, 0.0002998670700130344], "gradient_norm": 0.5069957375526428, "seconds": 0.23324879200004034, "nonpadding_tokens_per_second": 187743.73759668786, "ranks": [{"compute_seconds": 0.22945900700000266, "nonpadding_tokens": 8795, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23018142600000147, "nonpadding_tokens": 6605, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22714352799999915, "nonpadding_tokens": 10366, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22904578099996797, "nonpadding_tokens": 4325, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23061361999998553, "nonpadding_tokens": 1482, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22886100199997372, "nonpadding_tokens": 2718, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.22580764000002773, "nonpadding_tokens": 4496, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23095352699999694, "nonpadding_tokens": 5004, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003565530000173567, "prefetch": {"data_wait_seconds": 0.0051564149997744835, "consumed_batches": 549, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1099, "tokens": 90426222, "loss_tokens": 74504, "loss": 2.9336845588827445, "lr": 0.00029986678835879044, "optimizer_lrs": [0.019991119223919364, 0.00029986678835879044], "gradient_norm": 0.41561493277549744, "seconds": 0.29702513399996633, "nonpadding_tokens_per_second": 250833.99171198907, "ranks": [{"compute_seconds": 0.29477110199997014, "nonpadding_tokens": 13203, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2905356050000023, "nonpadding_tokens": 5558, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2894584899999586, "nonpadding_tokens": 11580, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28985492999998996, "nonpadding_tokens": 6943, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2884540229999857, "nonpadding_tokens": 7365, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29152270200000885, "nonpadding_tokens": 2426, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2941303810000022, "nonpadding_tokens": 6894, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2896208220000176, "nonpadding_tokens": 20535, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035920800002031683, "prefetch": {"data_wait_seconds": 0.005164585999764881, "consumed_batches": 550, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1100, "tokens": 90503200, "loss_tokens": 76978, "loss": 3.2301712827041493, "lr": 0.00029986649703893185, "optimizer_lrs": [0.019991099802595457, 0.00029986649703893185], "gradient_norm": 1.63884699344635, "seconds": 0.3529947380000067, "nonpadding_tokens_per_second": 218071.2393508782, "ranks": [{"compute_seconds": 0.35025008100001287, "nonpadding_tokens": 19192, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34690003299999717, "nonpadding_tokens": 8079, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3451380970000173, "nonpadding_tokens": 2710, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34609739899997294, "nonpadding_tokens": 9820, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3444070919999831, "nonpadding_tokens": 11936, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3470867219999718, "nonpadding_tokens": 11497, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3500425489999657, "nonpadding_tokens": 2055, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35064584500003093, "nonpadding_tokens": 11689, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003917340000043623, "prefetch": {"data_wait_seconds": 0.00517365999981223, "consumed_batches": 551, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1101, "tokens": 90555078, "loss_tokens": 51878, "loss": 2.907835149292571, "lr": 0.0002998663005295283, "optimizer_lrs": [0.019991086701968558, 0.0002998663005295283], "gradient_norm": 0.4699114263057709, "seconds": 0.24517563399996334, "nonpadding_tokens_per_second": 211595.2517533115, "ranks": [{"compute_seconds": 0.20824887300000228, "nonpadding_tokens": 5397, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21577313900002082, "nonpadding_tokens": 2809, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21232418800002506, "nonpadding_tokens": 4485, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21534215700000914, "nonpadding_tokens": 9576, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2159982849999551, "nonpadding_tokens": 1939, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2130529120000233, "nonpadding_tokens": 9382, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.21593463900001097, "nonpadding_tokens": 8242, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21628983300001892, "nonpadding_tokens": 10048, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.006240044999969996, "prefetch": {"data_wait_seconds": 0.0051828649997816, "consumed_batches": 552, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1102, "tokens": 90606140, "loss_tokens": 51062, "loss": 3.102515385707571, "lr": 0.0002998661069700051, "optimizer_lrs": [0.01999107379800034, 0.0002998661069700051], "gradient_norm": 0.4928674101829529, "seconds": 0.2912512170000241, "nonpadding_tokens_per_second": 175319.43909438077, "ranks": [{"compute_seconds": 0.2880788770000322, "nonpadding_tokens": 3436, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28848478600002636, "nonpadding_tokens": 11187, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28732603599996764, "nonpadding_tokens": 15438, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2882315629999539, "nonpadding_tokens": 4166, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.285376403999976, "nonpadding_tokens": 3087, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28568908299996565, "nonpadding_tokens": 3070, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2882884370000056, "nonpadding_tokens": 7089, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28892201799999384, "nonpadding_tokens": 3589, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034280199997738237, "prefetch": {"data_wait_seconds": 0.005190528999733601, "consumed_batches": 553, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1103, "tokens": 90662915, "loss_tokens": 56775, "loss": 2.5157953544693967, "lr": 0.00029986589159004655, "optimizer_lrs": [0.019991059439336437, 0.00029986589159004655], "gradient_norm": 0.5821920037269592, "seconds": 0.2852749909999943, "nonpadding_tokens_per_second": 199018.49720854475, "ranks": [{"compute_seconds": 0.20702656299999944, "nonpadding_tokens": 5612, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21167214299998705, "nonpadding_tokens": 4184, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21416343900000356, "nonpadding_tokens": 17573, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21452662699999792, "nonpadding_tokens": 3363, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2153947250000101, "nonpadding_tokens": 11961, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21201737900003081, "nonpadding_tokens": 8858, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.20910838499997908, "nonpadding_tokens": 2327, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21002317599999287, "nonpadding_tokens": 2897, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.04120581199998696, "prefetch": {"data_wait_seconds": 0.0052004739997215665, "consumed_batches": 554, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1104, "tokens": 90719100, "loss_tokens": 56185, "loss": 2.7835621162231914, "lr": 0.0002998656782779774, "optimizer_lrs": [0.01999104521853183, 0.0002998656782779774], "gradient_norm": 0.40130165219306946, "seconds": 0.2700959900000157, "nonpadding_tokens_per_second": 208018.63811453377, "ranks": [{"compute_seconds": 0.26689704599999686, "nonpadding_tokens": 3860, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26615130499999395, "nonpadding_tokens": 10394, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2628982869999845, "nonpadding_tokens": 7496, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26740281700000423, "nonpadding_tokens": 2818, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2677770479999708, "nonpadding_tokens": 8485, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26451427100005276, "nonpadding_tokens": 9555, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26736389199999167, "nonpadding_tokens": 4930, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26233664199997975, "nonpadding_tokens": 8647, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034537099998033227, "prefetch": {"data_wait_seconds": 0.005209123999705412, "consumed_batches": 555, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1105, "tokens": 90766782, "loss_tokens": 47682, "loss": 2.9690026503712095, "lr": 0.0002998654971155069, "optimizer_lrs": [0.019991033141033798, 0.0002998654971155069], "gradient_norm": 0.4800835847854614, "seconds": 0.275577434000013, "nonpadding_tokens_per_second": 173025.77830083776, "ranks": [{"compute_seconds": 0.27045463600001085, "nonpadding_tokens": 10062, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2728913330000182, "nonpadding_tokens": 6101, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.269828809000046, "nonpadding_tokens": 3069, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27273201200000585, "nonpadding_tokens": 7293, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2730051519999961, "nonpadding_tokens": 3458, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27035864900000206, "nonpadding_tokens": 7865, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2691947469999718, "nonpadding_tokens": 7448, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2697589649999941, "nonpadding_tokens": 2386, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0006025339999951029, "prefetch": {"data_wait_seconds": 0.005215165999700844, "consumed_batches": 556, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1106, "tokens": 90842490, "loss_tokens": 75708, "loss": 2.948175060759761, "lr": 0.0002998652092206699, "optimizer_lrs": [0.019991013948044662, 0.0002998652092206699], "gradient_norm": 0.7936389446258545, "seconds": 0.2620621339999616, "nonpadding_tokens_per_second": 288893.32023836416, "ranks": [{"compute_seconds": 0.259799973999975, "nonpadding_tokens": 3429, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.256238336000024, "nonpadding_tokens": 19511, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2588042630000018, "nonpadding_tokens": 10762, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2590412910000168, "nonpadding_tokens": 13985, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25439007100004574, "nonpadding_tokens": 4865, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25750221399999873, "nonpadding_tokens": 4896, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.25937424599999304, "nonpadding_tokens": 5245, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2550249330000156, "nonpadding_tokens": 13015, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033867999997028164, "prefetch": {"data_wait_seconds": 0.005223611999682021, "consumed_batches": 557, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1107, "tokens": 90911124, "loss_tokens": 68634, "loss": 2.667516737331352, "lr": 0.0002998649479602811, "optimizer_lrs": [0.019990996530685406, 0.0002998649479602811], "gradient_norm": 0.5959118008613586, "seconds": 0.3505977199999961, "nonpadding_tokens_per_second": 195762.82469834876, "ranks": [{"compute_seconds": 0.34734055900003113, "nonpadding_tokens": 11516, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3474261599999977, "nonpadding_tokens": 1529, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3470081390000246, "nonpadding_tokens": 12103, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34501240200000893, "nonpadding_tokens": 13420, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34824872399997275, "nonpadding_tokens": 8090, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3460283759999925, "nonpadding_tokens": 11910, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.34766856900000676, "nonpadding_tokens": 4687, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34370407300002626, "nonpadding_tokens": 5379, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034972800000332427, "prefetch": {"data_wait_seconds": 0.00523316999966994, "consumed_batches": 558, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1108, "tokens": 90988251, "loss_tokens": 77127, "loss": 2.7887228613196418, "lr": 0.0002998646540689728, "optimizer_lrs": [0.01999097693793152, 0.0002998646540689728], "gradient_norm": 1.0997552871704102, "seconds": 0.3806450979999454, "nonpadding_tokens_per_second": 202621.81335121533, "ranks": [{"compute_seconds": 0.3698915349999652, "nonpadding_tokens": 12112, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37732883199998923, "nonpadding_tokens": 4246, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37645445199996175, "nonpadding_tokens": 9632, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.377281537999977, "nonpadding_tokens": 13477, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3778240299999993, "nonpadding_tokens": 10837, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37640242000003354, "nonpadding_tokens": 4510, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3775963420000039, "nonpadding_tokens": 3213, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3783191330000477, "nonpadding_tokens": 19100, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040402799999128547, "prefetch": {"data_wait_seconds": 0.005258552999649169, "consumed_batches": 559, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1109, "tokens": 91047616, "loss_tokens": 59365, "loss": 3.1619346416238523, "lr": 0.0002998644276420626, "optimizer_lrs": [0.019990961842804175, 0.0002998644276420626], "gradient_norm": 0.4895249307155609, "seconds": 0.3262836289999882, "nonpadding_tokens_per_second": 181942.9316203975, "ranks": [{"compute_seconds": 0.3237766200000465, "nonpadding_tokens": 12443, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.320356439999955, "nonpadding_tokens": 8401, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32293189399996436, "nonpadding_tokens": 4684, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31975068399998463, "nonpadding_tokens": 4852, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32393672900002457, "nonpadding_tokens": 6550, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32106041100001903, "nonpadding_tokens": 4411, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3235935249999784, "nonpadding_tokens": 4715, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31867911300003016, "nonpadding_tokens": 13309, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040543999995179547, "prefetch": {"data_wait_seconds": 0.005269208999621355, "consumed_batches": 560, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1110, "tokens": 91095228, "loss_tokens": 47612, "loss": 3.0426418891245905, "lr": 0.0002998642459061534, "optimizer_lrs": [0.019990949727076896, 0.0002998642459061534], "gradient_norm": 0.4614868760108948, "seconds": 0.31492991599998277, "nonpadding_tokens_per_second": 151182.84285193981, "ranks": [{"compute_seconds": 0.31178976500001454, "nonpadding_tokens": 8331, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30846656299996766, "nonpadding_tokens": 7981, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3087761090000072, "nonpadding_tokens": 9158, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3115191560000312, "nonpadding_tokens": 7403, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31218087600001354, "nonpadding_tokens": 1774, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30944289500001787, "nonpadding_tokens": 3913, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.30806402100000696, "nonpadding_tokens": 6222, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3084341929999823, "nonpadding_tokens": 2830, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041393300000436284, "prefetch": {"data_wait_seconds": 0.0052787489996291015, "consumed_batches": 561, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1111, "tokens": 91165360, "loss_tokens": 70132, "loss": 2.764982016055438, "lr": 0.00029986397798937957, "optimizer_lrs": [0.019990931865958638, 0.00029986397798937957], "gradient_norm": 0.4280185103416443, "seconds": 0.25731309100001454, "nonpadding_tokens_per_second": 272555.1184645947, "ranks": [{"compute_seconds": 0.2536033639999573, "nonpadding_tokens": 1709, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25496348199999375, "nonpadding_tokens": 13376, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25353677900000093, "nonpadding_tokens": 4703, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2539257439999574, "nonpadding_tokens": 1994, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25457445299997516, "nonpadding_tokens": 6760, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2533548069999938, "nonpadding_tokens": 4465, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2504504519999955, "nonpadding_tokens": 25010, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25099833199999466, "nonpadding_tokens": 12115, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003368359999740278, "prefetch": {"data_wait_seconds": 0.005287088999651246, "consumed_batches": 562, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1112, "tokens": 91286062, "loss_tokens": 120702, "loss": 2.6890158717336914, "lr": 0.000299863516268042, "optimizer_lrs": [0.019990901084536133, 0.000299863516268042], "gradient_norm": 0.3493664264678955, "seconds": 0.32022202500002095, "nonpadding_tokens_per_second": 376932.22382186895, "ranks": [{"compute_seconds": 0.3175902070000234, "nonpadding_tokens": 11676, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3131968870000037, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3128248259999964, "nonpadding_tokens": 12125, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31685901600002353, "nonpadding_tokens": 7108, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3119189679999863, "nonpadding_tokens": 9400, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3146652589999803, "nonpadding_tokens": 8442, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3116103489999773, "nonpadding_tokens": 27669, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3125136909999924, "nonpadding_tokens": 11514, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036902599998711594, "prefetch": {"data_wait_seconds": 0.005295481999624485, "consumed_batches": 563, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1113, "tokens": 91363802, "loss_tokens": 77740, "loss": 2.772684388667353, "lr": 0.0002998632184752505, "optimizer_lrs": [0.01999088123168337, 0.0002998632184752505], "gradient_norm": 0.4365551769733429, "seconds": 0.3455162239999936, "nonpadding_tokens_per_second": 224996.67048920237, "ranks": [{"compute_seconds": 0.34083952999998246, "nonpadding_tokens": 8910, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3389530520000221, "nonpadding_tokens": 21776, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3393062559999862, "nonpadding_tokens": 11173, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3400765200000251, "nonpadding_tokens": 1846, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33879138000003195, "nonpadding_tokens": 7583, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3416329210000413, "nonpadding_tokens": 16936, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3385585550000201, "nonpadding_tokens": 3709, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3429597710000394, "nonpadding_tokens": 5807, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004676849999896149, "prefetch": {"data_wait_seconds": 0.005305740999588693, "consumed_batches": 564, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1114, "tokens": 91436107, "loss_tokens": 72305, "loss": 3.119947185533504, "lr": 0.0002998629412107712, "optimizer_lrs": [0.019990862747384746, 0.0002998629412107712], "gradient_norm": 0.44444841146469116, "seconds": 0.27678123300000834, "nonpadding_tokens_per_second": 261235.1972577484, "ranks": [{"compute_seconds": 0.2745158559999936, "nonpadding_tokens": 6793, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2704810820000034, "nonpadding_tokens": 5938, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2733408850000387, "nonpadding_tokens": 9590, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27168185699997593, "nonpadding_tokens": 19366, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2701185399999986, "nonpadding_tokens": 5448, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2726311870000018, "nonpadding_tokens": 9534, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2701563109999938, "nonpadding_tokens": 4499, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2707367210000484, "nonpadding_tokens": 11137, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034726100000170845, "prefetch": {"data_wait_seconds": 0.005314962999605655, "consumed_batches": 565, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1115, "tokens": 91518153, "loss_tokens": 82046, "loss": 2.697408694512834, "lr": 0.0002998626262531143, "optimizer_lrs": [0.01999084175020762, 0.0002998626262531143], "gradient_norm": 0.489321231842041, "seconds": 0.2967266949999612, "nonpadding_tokens_per_second": 276503.6020773619, "ranks": [{"compute_seconds": 0.29421659799999134, "nonpadding_tokens": 13010, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2897317890000295, "nonpadding_tokens": 9395, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29202253500000097, "nonpadding_tokens": 15500, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29293530299997883, "nonpadding_tokens": 17954, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28948250799999187, "nonpadding_tokens": 3362, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2918368069999815, "nonpadding_tokens": 2079, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2892698100000075, "nonpadding_tokens": 13223, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29399619399998755, "nonpadding_tokens": 7523, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038324499996633676, "prefetch": {"data_wait_seconds": 0.005324808999617403, "consumed_batches": 566, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1116, "tokens": 91592001, "loss_tokens": 73848, "loss": 2.7329857443668075, "lr": 0.0002998623424569495, "optimizer_lrs": [0.0199908228304633, 0.0002998623424569495], "gradient_norm": 0.471015602350235, "seconds": 0.27998390999999856, "nonpadding_tokens_per_second": 263758.0138087234, "ranks": [{"compute_seconds": 0.2760344499999974, "nonpadding_tokens": 3165, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.273871363000012, "nonpadding_tokens": 13333, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27298998499998106, "nonpadding_tokens": 9844, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27322711300001856, "nonpadding_tokens": 8899, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27160015399999793, "nonpadding_tokens": 19892, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2747835740000255, "nonpadding_tokens": 11966, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27712139700003036, "nonpadding_tokens": 2360, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2776055230000338, "nonpadding_tokens": 4389, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000408580999987862, "prefetch": {"data_wait_seconds": 0.0053336759996369665, "consumed_batches": 567, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1117, "tokens": 91659067, "loss_tokens": 67066, "loss": 2.6524018783735426, "lr": 0.0002998620844702493, "optimizer_lrs": [0.019990805631349955, 0.0002998620844702493], "gradient_norm": 0.627348780632019, "seconds": 0.28554145300000755, "nonpadding_tokens_per_second": 234873.07813061462, "ranks": [{"compute_seconds": 0.279073961999984, "nonpadding_tokens": 6672, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2787287660000288, "nonpadding_tokens": 2700, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2815339140000219, "nonpadding_tokens": 10816, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28184631999999965, "nonpadding_tokens": 17421, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2825826920000054, "nonpadding_tokens": 7622, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28039595500001724, "nonpadding_tokens": 4388, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27538766100002476, "nonpadding_tokens": 4272, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28305920299999343, "nonpadding_tokens": 13175, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004254209999885461, "prefetch": {"data_wait_seconds": 0.005343909999638186, "consumed_batches": 568, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1118, "tokens": 91728344, "loss_tokens": 69277, "loss": 2.7691006304401173, "lr": 0.0002998618177248984, "optimizer_lrs": [0.01999078784832656, 0.0002998618177248984], "gradient_norm": 0.5371212363243103, "seconds": 0.26465422600000466, "nonpadding_tokens_per_second": 261764.19340456245, "ranks": [{"compute_seconds": 0.25853458300002785, "nonpadding_tokens": 9013, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2583504079999557, "nonpadding_tokens": 10952, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26084139699997877, "nonpadding_tokens": 4797, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26165069899997206, "nonpadding_tokens": 15212, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26209956599996076, "nonpadding_tokens": 999, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.259294892000014, "nonpadding_tokens": 2041, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26168345800005, "nonpadding_tokens": 17837, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2621717649999482, "nonpadding_tokens": 8426, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042635500000187676, "prefetch": {"data_wait_seconds": 0.005353684999661823, "consumed_batches": 569, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1119, "tokens": 91782148, "loss_tokens": 53804, "loss": 2.884949306742993, "lr": 0.00029986161037934495, "optimizer_lrs": [0.019990774025289666, 0.00029986161037934495], "gradient_norm": 0.44896507263183594, "seconds": 0.2524504840000077, "nonpadding_tokens_per_second": 213126.94334148458, "ranks": [{"compute_seconds": 0.24665804500000377, "nonpadding_tokens": 4136, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24519083300003786, "nonpadding_tokens": 9883, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2424791199999845, "nonpadding_tokens": 5464, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2430870059999961, "nonpadding_tokens": 8265, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25009269099996345, "nonpadding_tokens": 9857, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24725811899998007, "nonpadding_tokens": 5149, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24138488500000221, "nonpadding_tokens": 2977, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24206300000003012, "nonpadding_tokens": 8073, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033306199998150987, "prefetch": {"data_wait_seconds": 0.005361652999638977, "consumed_batches": 570, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1120, "tokens": 91843030, "loss_tokens": 60882, "loss": 3.0660052437502054, "lr": 0.0002998613755697917, "optimizer_lrs": [0.019990758371319448, 0.0002998613755697917], "gradient_norm": 0.5454888939857483, "seconds": 0.3250863349999804, "nonpadding_tokens_per_second": 187279.480695501, "ranks": [{"compute_seconds": 0.32190148800003726, "nonpadding_tokens": 1647, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32222420900001225, "nonpadding_tokens": 2838, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32171580700003233, "nonpadding_tokens": 11342, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3224254149999979, "nonpadding_tokens": 15532, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31849144300002763, "nonpadding_tokens": 3680, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32017818100001705, "nonpadding_tokens": 4093, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.32266051799996376, "nonpadding_tokens": 10614, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31921506899999486, "nonpadding_tokens": 11136, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040715199997976015, "prefetch": {"data_wait_seconds": 0.005370068999596356, "consumed_batches": 571, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1121, "tokens": 91902865, "loss_tokens": 59835, "loss": 3.1686750543160356, "lr": 0.00029986114460447494, "optimizer_lrs": [0.019990742973631664, 0.00029986114460447494], "gradient_norm": 0.6420632004737854, "seconds": 0.2931590210000081, "nonpadding_tokens_per_second": 204104.24279592044, "ranks": [{"compute_seconds": 0.28652891299998373, "nonpadding_tokens": 9378, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2877491060000352, "nonpadding_tokens": 3432, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2853838959999848, "nonpadding_tokens": 6006, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2860840289999942, "nonpadding_tokens": 12118, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29083260200002314, "nonpadding_tokens": 11746, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2880611859999931, "nonpadding_tokens": 3672, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2841785800000025, "nonpadding_tokens": 3966, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2851367900000241, "nonpadding_tokens": 9517, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003517400000419002, "prefetch": {"data_wait_seconds": 0.005379154999573643, "consumed_batches": 572, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1122, "tokens": 91955759, "loss_tokens": 52894, "loss": 2.7494558692857414, "lr": 0.0002998609402716679, "optimizer_lrs": [0.01999072935144453, 0.0002998609402716679], "gradient_norm": 0.7649916410446167, "seconds": 0.26406842700004063, "nonpadding_tokens_per_second": 200304.14313783852, "ranks": [{"compute_seconds": 0.2615495069999838, "nonpadding_tokens": 10350, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25705576299998256, "nonpadding_tokens": 5472, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2556181730000162, "nonpadding_tokens": 6298, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25631117700004324, "nonpadding_tokens": 8823, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26096351199998935, "nonpadding_tokens": 6429, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2580391409999834, "nonpadding_tokens": 8154, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26013370300000815, "nonpadding_tokens": 2696, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2550953150000055, "nonpadding_tokens": 4672, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003330440000013368, "prefetch": {"data_wait_seconds": 0.0053877119995746625, "consumed_batches": 573, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1123, "tokens": 92006564, "loss_tokens": 50805, "loss": 2.752661216907785, "lr": 0.0002998607438674208, "optimizer_lrs": [0.019990716257828053, 0.0002998607438674208], "gradient_norm": 0.442162424325943, "seconds": 0.23712870900004646, "nonpadding_tokens_per_second": 214250.73418668192, "ranks": [{"compute_seconds": 0.23462147899999763, "nonpadding_tokens": 3252, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23065774899998814, "nonpadding_tokens": 11516, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23065756400001192, "nonpadding_tokens": 10812, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23078845399999182, "nonpadding_tokens": 3330, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2294147179999868, "nonpadding_tokens": 6611, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23199909400000251, "nonpadding_tokens": 3669, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23425934900001266, "nonpadding_tokens": 6961, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23005992700001343, "nonpadding_tokens": 4654, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004165309999848432, "prefetch": {"data_wait_seconds": 0.005396719999581023, "consumed_batches": 574, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1124, "tokens": 92088798, "loss_tokens": 82234, "loss": 2.666884287520977, "lr": 0.0002998604256699559, "optimizer_lrs": [0.01999069504466373, 0.0002998604256699559], "gradient_norm": 0.6320317983627319, "seconds": 0.24200686799997584, "nonpadding_tokens_per_second": 339800.27376747096, "ranks": [{"compute_seconds": 0.2365344529999902, "nonpadding_tokens": 9565, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23510835500002258, "nonpadding_tokens": 17926, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23803969299996197, "nonpadding_tokens": 1952, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23853340800002343, "nonpadding_tokens": 6720, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23527596000002404, "nonpadding_tokens": 10456, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23625682800002323, "nonpadding_tokens": 5299, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23523872699996673, "nonpadding_tokens": 18847, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23963851000002023, "nonpadding_tokens": 11469, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037504400000898386, "prefetch": {"data_wait_seconds": 0.005405314999578081, "consumed_batches": 575, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1125, "tokens": 92150983, "loss_tokens": 62185, "loss": 2.9589372939615664, "lr": 0.00029986018480939673, "optimizer_lrs": [0.019990678987293117, 0.00029986018480939673], "gradient_norm": 0.48712244629859924, "seconds": 0.3211776619999682, "nonpadding_tokens_per_second": 193615.58214470767, "ranks": [{"compute_seconds": 0.3178404980000096, "nonpadding_tokens": 2057, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3151348060000032, "nonpadding_tokens": 9076, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3173758169999701, "nonpadding_tokens": 9372, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3179332210000325, "nonpadding_tokens": 3746, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3187720700000227, "nonpadding_tokens": 9899, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31766748600000483, "nonpadding_tokens": 4446, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31822847500001217, "nonpadding_tokens": 7902, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3151146299999823, "nonpadding_tokens": 15687, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037235900003906863, "prefetch": {"data_wait_seconds": 0.005414904999568648, "consumed_batches": 576, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1126, "tokens": 92208766, "loss_tokens": 57783, "loss": 2.720338648477926, "lr": 0.00029985996081304125, "optimizer_lrs": [0.01999066405420275, 0.00029985996081304125], "gradient_norm": 0.4911128580570221, "seconds": 0.24888182600000164, "nonpadding_tokens_per_second": 232170.42774348505, "ranks": [{"compute_seconds": 0.24574036100000285, "nonpadding_tokens": 1233, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24532187900001645, "nonpadding_tokens": 3124, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24541070200001514, "nonpadding_tokens": 11304, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24260900000001584, "nonpadding_tokens": 9283, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24598375900001201, "nonpadding_tokens": 6588, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24346028299999034, "nonpadding_tokens": 7006, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24568359199997758, "nonpadding_tokens": 9468, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24648989299998902, "nonpadding_tokens": 9777, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000403156000004401, "prefetch": {"data_wait_seconds": 0.00542440799956978, "consumed_batches": 577, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1127, "tokens": 92269188, "loss_tokens": 60422, "loss": 3.180657500579259, "lr": 0.0002998597263949197, "optimizer_lrs": [0.019990648426327983, 0.0002998597263949197], "gradient_norm": 0.5576621890068054, "seconds": 0.3378002409999681, "nonpadding_tokens_per_second": 178869.0257328908, "ranks": [{"compute_seconds": 0.2798302179999723, "nonpadding_tokens": 11890, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2897029569999745, "nonpadding_tokens": 12831, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28839212400004044, "nonpadding_tokens": 10166, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2875305860000026, "nonpadding_tokens": 2185, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2858541649999893, "nonpadding_tokens": 9678, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28693501899999774, "nonpadding_tokens": 4500, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2860511769999903, "nonpadding_tokens": 3865, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2901975340000149, "nonpadding_tokens": 5307, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.026672636000000693, "prefetch": {"data_wait_seconds": 0.005433024999561553, "consumed_batches": 578, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1128, "tokens": 92309556, "loss_tokens": 40368, "loss": 2.9601494376734045, "lr": 0.0002998595696707517, "optimizer_lrs": [0.019990637978050115, 0.0002998595696707517], "gradient_norm": 1.3294895887374878, "seconds": 0.32183913699998357, "nonpadding_tokens_per_second": 125429.1208219405, "ranks": [{"compute_seconds": 0.31722164600000724, "nonpadding_tokens": 6346, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31849795900001254, "nonpadding_tokens": 2122, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3161680009999941, "nonpadding_tokens": 1457, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3165536940000493, "nonpadding_tokens": 6349, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3187967949999688, "nonpadding_tokens": 4636, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31751028200000064, "nonpadding_tokens": 1994, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31854377700000214, "nonpadding_tokens": 6343, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31945785599998544, "nonpadding_tokens": 11121, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040806000004067755, "prefetch": {"data_wait_seconds": 0.005453610999566649, "consumed_batches": 579, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1129, "tokens": 92387190, "loss_tokens": 77634, "loss": 2.927568381765721, "lr": 0.0002998592680197483, "optimizer_lrs": [0.019990617867983222, 0.0002998592680197483], "gradient_norm": 0.4383948743343353, "seconds": 0.2999130910000076, "nonpadding_tokens_per_second": 258854.98942758061, "ranks": [{"compute_seconds": 0.2967667720000122, "nonpadding_tokens": 16344, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29671586900002467, "nonpadding_tokens": 6536, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29389348200004406, "nonpadding_tokens": 4568, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29689201000002186, "nonpadding_tokens": 7669, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29313807899995936, "nonpadding_tokens": 6303, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29453069499999174, "nonpadding_tokens": 4495, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2970703440000193, "nonpadding_tokens": 19729, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2975813090000088, "nonpadding_tokens": 11990, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039012500002399975, "prefetch": {"data_wait_seconds": 0.005462647999550541, "consumed_batches": 580, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1130, "tokens": 92430090, "loss_tokens": 42900, "loss": 2.706955128205128, "lr": 0.00029985910119077644, "optimizer_lrs": [0.019990606746051763, 0.00029985910119077644], "gradient_norm": 0.5242166519165039, "seconds": 0.2971837420000156, "nonpadding_tokens_per_second": 144355.13770466537, "ranks": [{"compute_seconds": 0.29401390999998966, "nonpadding_tokens": 6218, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2949581450000096, "nonpadding_tokens": 5141, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2920517489999952, "nonpadding_tokens": 3522, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2946773950000079, "nonpadding_tokens": 2525, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29079024200001413, "nonpadding_tokens": 8636, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29209826300001396, "nonpadding_tokens": 10116, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.29450698100004047, "nonpadding_tokens": 2774, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2912238359999719, "nonpadding_tokens": 3968, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034641799999235445, "prefetch": {"data_wait_seconds": 0.005470777999576626, "consumed_batches": 581, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1131, "tokens": 92501658, "loss_tokens": 71568, "loss": 3.06176730522021, "lr": 0.00029985882265821874, "optimizer_lrs": [0.019990588177214583, 0.00029985882265821874], "gradient_norm": 0.5364655256271362, "seconds": 0.37799740899998824, "nonpadding_tokens_per_second": 189334.6311270674, "ranks": [{"compute_seconds": 0.3742413290000286, "nonpadding_tokens": 18353, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37160286200003156, "nonpadding_tokens": 8691, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3739029829999936, "nonpadding_tokens": 6826, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.37260043199995607, "nonpadding_tokens": 1796, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.37090985899999396, "nonpadding_tokens": 4989, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37235569099999566, "nonpadding_tokens": 10941, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3750368069999581, "nonpadding_tokens": 9934, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3756454089999579, "nonpadding_tokens": 10038, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034757100002025254, "prefetch": {"data_wait_seconds": 0.00547987399961869, "consumed_batches": 582, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1132, "tokens": 92586127, "loss_tokens": 84469, "loss": 2.633384177923262, "lr": 0.00029985849356310544, "optimizer_lrs": [0.019990566237540364, 0.00029985849356310544], "gradient_norm": 0.46023958921432495, "seconds": 0.32076767899997094, "nonpadding_tokens_per_second": 263333.8878260476, "ranks": [{"compute_seconds": 0.31641318200001933, "nonpadding_tokens": 11280, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31436902299998337, "nonpadding_tokens": 10306, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3175404360000016, "nonpadding_tokens": 14784, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31596155599999065, "nonpadding_tokens": 6449, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3183505629999672, "nonpadding_tokens": 12446, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3173742070000003, "nonpadding_tokens": 9356, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3143683509999846, "nonpadding_tokens": 14626, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3184538109999835, "nonpadding_tokens": 5222, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039094300001352167, "prefetch": {"data_wait_seconds": 0.005490440999565749, "consumed_batches": 583, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1133, "tokens": 92647113, "loss_tokens": 60986, "loss": 2.836033321172072, "lr": 0.00029985825572080235, "optimizer_lrs": [0.019990550381386824, 0.00029985825572080235], "gradient_norm": 0.4099853038787842, "seconds": 0.4012862269999573, "nonpadding_tokens_per_second": 151976.30991707693, "ranks": [{"compute_seconds": 0.3504061350000143, "nonpadding_tokens": 1385, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37751458699995055, "nonpadding_tokens": 5673, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3779066139999827, "nonpadding_tokens": 7421, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.38085093800003733, "nonpadding_tokens": 5428, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3814077859999543, "nonpadding_tokens": 17864, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37836640099999386, "nonpadding_tokens": 12731, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3807562789999679, "nonpadding_tokens": 4376, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3771195830000238, "nonpadding_tokens": 6108, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00854404199998271, "prefetch": {"data_wait_seconds": 0.005499115999569426, "consumed_batches": 584, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1134, "tokens": 92726551, "loss_tokens": 79438, "loss": 2.6280515150180013, "lr": 0.00029985794561730077, "optimizer_lrs": [0.019990529707820053, 0.00029985794561730077], "gradient_norm": 0.6008325815200806, "seconds": 0.30441342900002155, "nonpadding_tokens_per_second": 260954.3220906801, "ranks": [{"compute_seconds": 0.30111076999997977, "nonpadding_tokens": 14773, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29782475000001796, "nonpadding_tokens": 8684, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29829148599998234, "nonpadding_tokens": 8623, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2987157479999496, "nonpadding_tokens": 5314, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3014199950000034, "nonpadding_tokens": 14268, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29859774800002015, "nonpadding_tokens": 14544, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.29717923800001245, "nonpadding_tokens": 9316, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3020321970000168, "nonpadding_tokens": 3916, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038632700000107434, "prefetch": {"data_wait_seconds": 0.005510518999543024, "consumed_batches": 585, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1135, "tokens": 92808042, "loss_tokens": 81491, "loss": 2.8155150952252397, "lr": 0.00029985762714755746, "optimizer_lrs": [0.019990508476503834, 0.00029985762714755746], "gradient_norm": 0.3882027268409729, "seconds": 0.2450986179999859, "nonpadding_tokens_per_second": 332482.49486255646, "ranks": [{"compute_seconds": 0.23823142000003372, "nonpadding_tokens": 4561, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2380622310000149, "nonpadding_tokens": 11720, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23695284699999775, "nonpadding_tokens": 12844, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23748177999999598, "nonpadding_tokens": 8967, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2419035060000283, "nonpadding_tokens": 25881, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2390260939999962, "nonpadding_tokens": 5446, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23573218400002816, "nonpadding_tokens": 7611, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24280824000004486, "nonpadding_tokens": 4461, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003882060000250931, "prefetch": {"data_wait_seconds": 0.005520394999521727, "consumed_batches": 586, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1136, "tokens": 92856101, "loss_tokens": 48059, "loss": 2.937709378056139, "lr": 0.000299857439164196, "optimizer_lrs": [0.019990495944279735, 0.000299857439164196], "gradient_norm": 0.4396614134311676, "seconds": 0.367904464999981, "nonpadding_tokens_per_second": 130629.02077038528, "ranks": [{"compute_seconds": 0.32432538599999816, "nonpadding_tokens": 2374, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.361297325999999, "nonpadding_tokens": 6253, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36036263599999074, "nonpadding_tokens": 6645, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.36042821399996683, "nonpadding_tokens": 5433, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3571976549999931, "nonpadding_tokens": 4055, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35988921300003085, "nonpadding_tokens": 6114, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3610109800000032, "nonpadding_tokens": 6027, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36154680499998904, "nonpadding_tokens": 11158, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003879910000250675, "prefetch": {"data_wait_seconds": 0.005534283999523382, "consumed_batches": 587, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1137, "tokens": 92951058, "loss_tokens": 94957, "loss": 2.6704166096233033, "lr": 0.0002998570673743227, "optimizer_lrs": [0.01999047115828818, 0.0002998570673743227], "gradient_norm": 0.4116356074810028, "seconds": 0.25324483299999656, "nonpadding_tokens_per_second": 374961.2534049265, "ranks": [{"compute_seconds": 0.2498374520000084, "nonpadding_tokens": 6799, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24871553200000562, "nonpadding_tokens": 15234, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24946858800001337, "nonpadding_tokens": 19378, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2465690630000381, "nonpadding_tokens": 12921, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25093443900004786, "nonpadding_tokens": 1988, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24809063899999728, "nonpadding_tokens": 4911, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.25007547900003146, "nonpadding_tokens": 6067, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24519524900000533, "nonpadding_tokens": 27659, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000369615000010981, "prefetch": {"data_wait_seconds": 0.005543573999560181, "consumed_batches": 588, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1138, "tokens": 93018195, "loss_tokens": 67137, "loss": 2.7230381067816554, "lr": 0.0002998568042174608, "optimizer_lrs": [0.019990453614497385, 0.0002998568042174608], "gradient_norm": 0.5018442273139954, "seconds": 0.41357550999998693, "nonpadding_tokens_per_second": 162333.1130027552, "ranks": [{"compute_seconds": 0.4084247449999907, "nonpadding_tokens": 21830, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.40798526400004675, "nonpadding_tokens": 3478, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.40850992800000085, "nonpadding_tokens": 4703, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.41105230100004064, "nonpadding_tokens": 12890, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.40774636499997996, "nonpadding_tokens": 4202, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41004958100000977, "nonpadding_tokens": 8969, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.4112322170000198, "nonpadding_tokens": 8857, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.40826617400000487, "nonpadding_tokens": 2208, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003882480000356736, "prefetch": {"data_wait_seconds": 0.005552521999575788, "consumed_batches": 589, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1139, "tokens": 93078118, "loss_tokens": 59923, "loss": 2.814191232915575, "lr": 0.0002998565691330022, "optimizer_lrs": [0.019990437942200148, 0.0002998565691330022], "gradient_norm": 0.4858613610267639, "seconds": 0.2225931849999938, "nonpadding_tokens_per_second": 269204.1088320007, "ranks": [{"compute_seconds": 0.2190273180000304, "nonpadding_tokens": 4313, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2177415229999724, "nonpadding_tokens": 9860, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21881522700005007, "nonpadding_tokens": 9044, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2154913059999899, "nonpadding_tokens": 7451, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2195116759999678, "nonpadding_tokens": 6736, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2169330190000096, "nonpadding_tokens": 3262, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.21897011499999053, "nonpadding_tokens": 6180, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21504843200000323, "nonpadding_tokens": 13077, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004040030000282968, "prefetch": {"data_wait_seconds": 0.005568924999593605, "consumed_batches": 590, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1140, "tokens": 93188478, "loss_tokens": 110360, "loss": 2.8374450661471546, "lr": 0.00029985613567442345, "optimizer_lrs": [0.019990409044961563, 0.00029985613567442345], "gradient_norm": 0.4818928837776184, "seconds": 0.2737480190000383, "nonpadding_tokens_per_second": 403144.4698782808, "ranks": [{"compute_seconds": 0.27075073299999985, "nonpadding_tokens": 14144, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27125663399999667, "nonpadding_tokens": 27738, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2701636790000066, "nonpadding_tokens": 3810, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26844098200001554, "nonpadding_tokens": 11752, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26717525200001546, "nonpadding_tokens": 5205, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26825057199999947, "nonpadding_tokens": 11663, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2668941270000005, "nonpadding_tokens": 17671, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2714514060000397, "nonpadding_tokens": 18377, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003557210000053601, "prefetch": {"data_wait_seconds": 0.005577028999610434, "consumed_batches": 591, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1141, "tokens": 93262929, "loss_tokens": 74451, "loss": 2.8132192230460302, "lr": 0.0002998558428856294, "optimizer_lrs": [0.01999038952570863, 0.0002998558428856294], "gradient_norm": 0.6869238018989563, "seconds": 0.3022852660000126, "nonpadding_tokens_per_second": 246293.84351137013, "ranks": [{"compute_seconds": 0.3000561400000379, "nonpadding_tokens": 1897, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2993293640000161, "nonpadding_tokens": 6729, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2987896320000232, "nonpadding_tokens": 5298, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29932352399998763, "nonpadding_tokens": 12500, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29990153300002476, "nonpadding_tokens": 3198, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29856512899999643, "nonpadding_tokens": 30988, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2996447130000206, "nonpadding_tokens": 2218, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29645778099995823, "nonpadding_tokens": 11623, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034060400002999813, "prefetch": {"data_wait_seconds": 0.00558531599962464, "consumed_batches": 592, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1142, "tokens": 93378383, "loss_tokens": 115454, "loss": 2.6394473123495072, "lr": 0.0002998553882585635, "optimizer_lrs": [0.019990359217237567, 0.0002998553882585635], "gradient_norm": 0.5311585068702698, "seconds": 0.3760669719999896, "nonpadding_tokens_per_second": 307003.82803093694, "ranks": [{"compute_seconds": 0.37279942200001415, "nonpadding_tokens": 15033, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3691541879999818, "nonpadding_tokens": 13069, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3681352329999754, "nonpadding_tokens": 7387, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.36862522000001263, "nonpadding_tokens": 3063, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3671437220000371, "nonpadding_tokens": 15378, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3699922129999891, "nonpadding_tokens": 23010, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.367089094999983, "nonpadding_tokens": 26411, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3737620530000072, "nonpadding_tokens": 12103, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003442470000436515, "prefetch": {"data_wait_seconds": 0.0055934829996004964, "consumed_batches": 593, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1143, "tokens": 93421374, "loss_tokens": 42991, "loss": 2.8086318211951338, "lr": 0.00029985521878868614, "optimizer_lrs": [0.019990347919245744, 0.00029985521878868614], "gradient_norm": 0.47339925169944763, "seconds": 0.2407727419999901, "nonpadding_tokens_per_second": 178554.26508371852, "ranks": [{"compute_seconds": 0.23736980300003552, "nonpadding_tokens": 2449, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23534307100004526, "nonpadding_tokens": 9095, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2368878030000019, "nonpadding_tokens": 4935, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23711567599997352, "nonpadding_tokens": 4246, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2314198509999983, "nonpadding_tokens": 6117, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23532522299996117, "nonpadding_tokens": 4325, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2384519409999939, "nonpadding_tokens": 7930, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23211375699997916, "nonpadding_tokens": 3894, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035134000000880405, "prefetch": {"data_wait_seconds": 0.005602805999629368, "consumed_batches": 594, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1144, "tokens": 93476860, "loss_tokens": 55486, "loss": 2.7501475597448004, "lr": 0.0002998549999170849, "optimizer_lrs": [0.01999033332780566, 0.0002998549999170849], "gradient_norm": 0.41810303926467896, "seconds": 0.2568741289999821, "nonpadding_tokens_per_second": 216004.62536265719, "ranks": [{"compute_seconds": 0.2536062849999894, "nonpadding_tokens": 3312, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25013886300001786, "nonpadding_tokens": 2358, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2511049820000153, "nonpadding_tokens": 7118, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25385207599998694, "nonpadding_tokens": 14800, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2540986999999859, "nonpadding_tokens": 8981, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25262905799996815, "nonpadding_tokens": 10239, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24974871099999518, "nonpadding_tokens": 6305, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2545318850000058, "nonpadding_tokens": 2373, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037531899999976304, "prefetch": {"data_wait_seconds": 0.005611760999627791, "consumed_batches": 595, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1145, "tokens": 93543510, "loss_tokens": 66650, "loss": 2.8286984246061517, "lr": 0.0002998547367892274, "optimizer_lrs": [0.019990315785948495, 0.0002998547367892274], "gradient_norm": 0.5399017333984375, "seconds": 0.31350412700004426, "nonpadding_tokens_per_second": 212596.88233702452, "ranks": [{"compute_seconds": 0.31083625899998424, "nonpadding_tokens": 11463, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3107978330000378, "nonpadding_tokens": 2604, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30986357400001907, "nonpadding_tokens": 6760, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30878720399999793, "nonpadding_tokens": 12043, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31114550500001315, "nonpadding_tokens": 4082, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30952707900001997, "nonpadding_tokens": 11305, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.30655456099998446, "nonpadding_tokens": 9075, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30724228200000425, "nonpadding_tokens": 9318, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039196199998059456, "prefetch": {"data_wait_seconds": 0.005620979999619635, "consumed_batches": 596, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1146, "tokens": 93604479, "loss_tokens": 60969, "loss": 3.125058175056176, "lr": 0.0002998544958806312, "optimizer_lrs": [0.019990299725375416, 0.0002998544958806312], "gradient_norm": 0.4351547658443451, "seconds": 0.23725891500004082, "nonpadding_tokens_per_second": 256972.43030884428, "ranks": [{"compute_seconds": 0.23282718900003374, "nonpadding_tokens": 2751, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2345105509999712, "nonpadding_tokens": 9791, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2318974849999904, "nonpadding_tokens": 2829, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23410878000004232, "nonpadding_tokens": 8415, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2344841850000421, "nonpadding_tokens": 10465, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2323538510000276, "nonpadding_tokens": 1868, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2344625439999959, "nonpadding_tokens": 10942, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2349186249999775, "nonpadding_tokens": 13908, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003824789999953282, "prefetch": {"data_wait_seconds": 0.005630654999620788, "consumed_batches": 597, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1147, "tokens": 93645642, "loss_tokens": 41163, "loss": 2.624689876223793, "lr": 0.00029985433311925365, "optimizer_lrs": [0.019990288874616912, 0.00029985433311925365], "gradient_norm": 0.4709678292274475, "seconds": 0.35889414500002204, "nonpadding_tokens_per_second": 114693.98588265482, "ranks": [{"compute_seconds": 0.35576286200000595, "nonpadding_tokens": 13176, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3563315519999719, "nonpadding_tokens": 4937, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3530953659999909, "nonpadding_tokens": 1274, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.353627514999971, "nonpadding_tokens": 5066, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3563478439999699, "nonpadding_tokens": 4443, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.355280538000045, "nonpadding_tokens": 2007, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.35214751899997054, "nonpadding_tokens": 5015, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3565426680000314, "nonpadding_tokens": 5245, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003669769999987693, "prefetch": {"data_wait_seconds": 0.0056401489996034115, "consumed_batches": 598, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1148, "tokens": 93708682, "loss_tokens": 63040, "loss": 2.4059287753807106, "lr": 0.00029985408367842243, "optimizer_lrs": [0.019990272245228164, 0.00029985408367842243], "gradient_norm": 0.5532889366149902, "seconds": 0.31500945199996977, "nonpadding_tokens_per_second": 200120.97922701712, "ranks": [{"compute_seconds": 0.31268080000000964, "nonpadding_tokens": 5771, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3116018380000014, "nonpadding_tokens": 3581, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30868742600000587, "nonpadding_tokens": 4058, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31123313899996674, "nonpadding_tokens": 9904, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3076702219999561, "nonpadding_tokens": 4181, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3085603990000436, "nonpadding_tokens": 17402, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31138973600002373, "nonpadding_tokens": 3616, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31193333300001314, "nonpadding_tokens": 14527, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003896480000094016, "prefetch": {"data_wait_seconds": 0.005649293999582028, "consumed_batches": 599, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1149, "tokens": 93777507, "loss_tokens": 68825, "loss": 2.7201916091536504, "lr": 0.0002998538111035923, "optimizer_lrs": [0.019990254073572824, 0.0002998538111035923], "gradient_norm": 0.45104601979255676, "seconds": 0.25418132499999047, "nonpadding_tokens_per_second": 270771.26928975835, "ranks": [{"compute_seconds": 0.25174780899999405, "nonpadding_tokens": 7745, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24793474000000515, "nonpadding_tokens": 2700, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24814940999999635, "nonpadding_tokens": 2528, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24868277700005592, "nonpadding_tokens": 13319, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2515325599999869, "nonpadding_tokens": 19037, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25011518899998464, "nonpadding_tokens": 2493, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.25120831899999985, "nonpadding_tokens": 10034, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2483526230000166, "nonpadding_tokens": 10969, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040190999999367705, "prefetch": {"data_wait_seconds": 0.005674933999571863, "consumed_batches": 600, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1150, "tokens": 93821704, "loss_tokens": 44197, "loss": 2.653003173292305, "lr": 0.0002998536359315621, "optimizer_lrs": [0.019990242395437476, 0.0002998536359315621], "gradient_norm": 0.4434175193309784, "seconds": 0.2602568519999977, "nonpadding_tokens_per_second": 169820.69697823128, "ranks": [{"compute_seconds": 0.25578645699999925, "nonpadding_tokens": 5396, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25777059099999633, "nonpadding_tokens": 6038, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2552526559999819, "nonpadding_tokens": 3235, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2575538300000062, "nonpadding_tokens": 5461, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25794480199999725, "nonpadding_tokens": 5898, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25678710799996907, "nonpadding_tokens": 6155, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2576940989999912, "nonpadding_tokens": 7537, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.254361834000008, "nonpadding_tokens": 4477, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037598999995225313, "prefetch": {"data_wait_seconds": 0.0056839339996059834, "consumed_batches": 601, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1151, "tokens": 93908020, "loss_tokens": 86316, "loss": 2.7768001442374532, "lr": 0.00029985329352123763, "optimizer_lrs": [0.01999021956808251, 0.00029985329352123763], "gradient_norm": 0.4667656421661377, "seconds": 0.32695084599998836, "nonpadding_tokens_per_second": 264002.9871646305, "ranks": [{"compute_seconds": 0.3245224140000005, "nonpadding_tokens": 17647, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3207205550000367, "nonpadding_tokens": 15277, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3217192039999759, "nonpadding_tokens": 8844, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32215843800003086, "nonpadding_tokens": 10599, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32039739500004316, "nonpadding_tokens": 3813, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32168777900000123, "nonpadding_tokens": 13162, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.32033665599999495, "nonpadding_tokens": 7507, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3211695029999646, "nonpadding_tokens": 9467, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003852870000287112, "prefetch": {"data_wait_seconds": 0.005693353999575379, "consumed_batches": 602, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1152, "tokens": 93966484, "loss_tokens": 58464, "loss": 2.4697597025519977, "lr": 0.0002998530613709763, "optimizer_lrs": [0.019990204091398425, 0.0002998530613709763], "gradient_norm": 0.483514666557312, "seconds": 0.3189958920000322, "nonpadding_tokens_per_second": 183275.0874421734, "ranks": [{"compute_seconds": 0.31655467900003487, "nonpadding_tokens": 18705, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31316776400001345, "nonpadding_tokens": 2640, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.315834609000035, "nonpadding_tokens": 4842, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3134431149999841, "nonpadding_tokens": 2474, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3117887909999695, "nonpadding_tokens": 12885, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3142377949999968, "nonpadding_tokens": 3714, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.316592025000034, "nonpadding_tokens": 3526, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31216749000003574, "nonpadding_tokens": 9678, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003865310000037425, "prefetch": {"data_wait_seconds": 0.005702981999561985, "consumed_batches": 603, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1153, "tokens": 94015285, "loss_tokens": 48801, "loss": 2.9663316965840862, "lr": 0.0002998528674503266, "optimizer_lrs": [0.019990191163355108, 0.0002998528674503266], "gradient_norm": 0.48727405071258545, "seconds": 0.2651703850000331, "nonpadding_tokens_per_second": 184036.38852805493, "ranks": [{"compute_seconds": 0.26135130199998, "nonpadding_tokens": 8565, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26104239000000007, "nonpadding_tokens": 4602, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2609874019999552, "nonpadding_tokens": 4233, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2586525579999943, "nonpadding_tokens": 2674, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2621687769999994, "nonpadding_tokens": 6719, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.259002401000032, "nonpadding_tokens": 3170, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2567968959999689, "nonpadding_tokens": 6207, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26274397200000976, "nonpadding_tokens": 12631, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040705099996785066, "prefetch": {"data_wait_seconds": 0.005715044999547558, "consumed_batches": 604, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1154, "tokens": 94085506, "loss_tokens": 70221, "loss": 2.712192577718916, "lr": 0.00029985258818870486, "optimizer_lrs": [0.01999017254591366, 0.00029985258818870486], "gradient_norm": 0.44402727484703064, "seconds": 0.38202731499995934, "nonpadding_tokens_per_second": 183811.4638478337, "ranks": [{"compute_seconds": 0.3787796199999889, "nonpadding_tokens": 12832, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3756727539999929, "nonpadding_tokens": 8990, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3762474129999873, "nonpadding_tokens": 3157, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.37922550400003274, "nonpadding_tokens": 6559, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.37540593599999283, "nonpadding_tokens": 4395, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3782316990000254, "nonpadding_tokens": 5414, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.37549122400002943, "nonpadding_tokens": 10148, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37964995700002646, "nonpadding_tokens": 18726, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039494999998623825, "prefetch": {"data_wait_seconds": 0.005723933999561837, "consumed_batches": 605, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1155, "tokens": 94138942, "loss_tokens": 53436, "loss": 2.946468087993113, "lr": 0.00029985237550197844, "optimizer_lrs": [0.019990158366798565, 0.00029985237550197844], "gradient_norm": 0.4666830897331238, "seconds": 0.3905191520000244, "nonpadding_tokens_per_second": 136833.23782285757, "ranks": [{"compute_seconds": 0.3870454469999913, "nonpadding_tokens": 12058, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3870787800000244, "nonpadding_tokens": 4011, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3839211540000065, "nonpadding_tokens": 6497, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.38653306800000564, "nonpadding_tokens": 5253, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3831233370000291, "nonpadding_tokens": 4541, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38402667099995824, "nonpadding_tokens": 5748, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.38294328400002087, "nonpadding_tokens": 3236, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3882128449999982, "nonpadding_tokens": 12092, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003515590000233715, "prefetch": {"data_wait_seconds": 0.0057336259995537375, "consumed_batches": 606, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1156, "tokens": 94196802, "loss_tokens": 57860, "loss": 2.824989468112686, "lr": 0.0002998521450339908, "optimizer_lrs": [0.019990143002266057, 0.0002998521450339908], "gradient_norm": 0.47984567284584045, "seconds": 0.3416438419999963, "nonpadding_tokens_per_second": 169357.6552156928, "ranks": [{"compute_seconds": 0.3383870680000314, "nonpadding_tokens": 18272, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33549420800000007, "nonpadding_tokens": 6484, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3382370819999778, "nonpadding_tokens": 6965, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33907207299995434, "nonpadding_tokens": 2982, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33155492200000936, "nonpadding_tokens": 3054, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33648314599997775, "nonpadding_tokens": 6952, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3315203490000158, "nonpadding_tokens": 3946, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3320066609999799, "nonpadding_tokens": 9205, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003845989999717858, "prefetch": {"data_wait_seconds": 0.005741520999507088, "consumed_batches": 607, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1157, "tokens": 94251197, "loss_tokens": 54395, "loss": 2.7871331808989797, "lr": 0.0002998519282039482, "optimizer_lrs": [0.01999012854692988, 0.0002998519282039482], "gradient_norm": 0.5913980007171631, "seconds": 0.3141626909999786, "nonpadding_tokens_per_second": 173142.77461420046, "ranks": [{"compute_seconds": 0.3106217620000393, "nonpadding_tokens": 3570, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31143990699996493, "nonpadding_tokens": 3020, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3104113429999984, "nonpadding_tokens": 8396, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3108356879999974, "nonpadding_tokens": 4752, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3072963790000358, "nonpadding_tokens": 3475, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3083555779999756, "nonpadding_tokens": 10028, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3069347220000509, "nonpadding_tokens": 14256, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31176994700001615, "nonpadding_tokens": 6898, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004197279999971215, "prefetch": {"data_wait_seconds": 0.005751825999496418, "consumed_batches": 608, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1158, "tokens": 94327815, "loss_tokens": 76618, "loss": 2.864778226069592, "lr": 0.0002998516225189518, "optimizer_lrs": [0.019990108167930124, 0.0002998516225189518], "gradient_norm": 0.41450735926628113, "seconds": 0.24620030299996642, "nonpadding_tokens_per_second": 311201.8915752937, "ranks": [{"compute_seconds": 0.2202489910000054, "nonpadding_tokens": 3874, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2200909050000064, "nonpadding_tokens": 10774, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22044376799999554, "nonpadding_tokens": 5083, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22134826599994994, "nonpadding_tokens": 7921, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.22360760299994809, "nonpadding_tokens": 13676, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22078334500002939, "nonpadding_tokens": 13610, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.22063635100005285, "nonpadding_tokens": 10847, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22028427200001488, "nonpadding_tokens": 10833, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.004371795999986716, "prefetch": {"data_wait_seconds": 0.0057604759994802635, "consumed_batches": 609, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1159, "tokens": 94411064, "loss_tokens": 83249, "loss": 3.0333823454335787, "lr": 0.0002998512900209551, "optimizer_lrs": [0.019990086001397008, 0.0002998512900209551], "gradient_norm": 0.4139919877052307, "seconds": 0.3303540739999562, "nonpadding_tokens_per_second": 251999.3139240385, "ranks": [{"compute_seconds": 0.3258515040000134, "nonpadding_tokens": 10984, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3277125359999786, "nonpadding_tokens": 20918, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3245266430000129, "nonpadding_tokens": 4626, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3276180479999766, "nonpadding_tokens": 2835, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3275999039999533, "nonpadding_tokens": 3892, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32669592400003467, "nonpadding_tokens": 18134, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.32385893700001134, "nonpadding_tokens": 9405, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32469255900002736, "nonpadding_tokens": 12455, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039149900004531446, "prefetch": {"data_wait_seconds": 0.005770350999526386, "consumed_batches": 610, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1160, "tokens": 94484464, "loss_tokens": 73400, "loss": 2.8897366740463215, "lr": 0.0002998509965515161, "optimizer_lrs": [0.01999006643676774, 0.0002998509965515161], "gradient_norm": 0.44426923990249634, "seconds": 0.2715040120000367, "nonpadding_tokens_per_second": 270345.9129730653, "ranks": [{"compute_seconds": 0.2686593020000032, "nonpadding_tokens": 6247, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26834333099998275, "nonpadding_tokens": 6174, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2654155669999909, "nonpadding_tokens": 1957, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26652006099999426, "nonpadding_tokens": 9929, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2687266820000218, "nonpadding_tokens": 9578, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2659188840000297, "nonpadding_tokens": 11757, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26877695700000004, "nonpadding_tokens": 23346, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26913631500002566, "nonpadding_tokens": 4412, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037849299997105845, "prefetch": {"data_wait_seconds": 0.005778934999526086, "consumed_batches": 611, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1161, "tokens": 94562344, "loss_tokens": 77880, "loss": 2.9140871773882897, "lr": 0.00029985068485391094, "optimizer_lrs": [0.019990045656927396, 0.00029985068485391094], "gradient_norm": 0.40270617604255676, "seconds": 0.26870218900000964, "nonpadding_tokens_per_second": 289837.6090266879, "ranks": [{"compute_seconds": 0.26564938999996457, "nonpadding_tokens": 4781, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2622700090000194, "nonpadding_tokens": 10057, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2625605350000342, "nonpadding_tokens": 9723, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2659535989999995, "nonpadding_tokens": 4618, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26172514400002456, "nonpadding_tokens": 9719, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26300765199999887, "nonpadding_tokens": 4855, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26174082299996826, "nonpadding_tokens": 17043, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26633697900001607, "nonpadding_tokens": 17084, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003687439999566777, "prefetch": {"data_wait_seconds": 0.005788440999552336, "consumed_batches": 612, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1162, "tokens": 94614029, "loss_tokens": 51685, "loss": 2.6747103850246687, "lr": 0.0002998504778163564, "optimizer_lrs": [0.019990031854423763, 0.0002998504778163564], "gradient_norm": 0.448451429605484, "seconds": 0.311916258999986, "nonpadding_tokens_per_second": 165701.52567776956, "ranks": [{"compute_seconds": 0.3076849199999856, "nonpadding_tokens": 15100, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3093342379999626, "nonpadding_tokens": 3020, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3084638459999951, "nonpadding_tokens": 4085, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30677568000004385, "nonpadding_tokens": 10207, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3053019909999648, "nonpadding_tokens": 6457, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.308321626999998, "nonpadding_tokens": 3251, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.30905133400000295, "nonpadding_tokens": 6082, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3096249109999576, "nonpadding_tokens": 3483, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035956799996483824, "prefetch": {"data_wait_seconds": 0.005798915999562837, "consumed_batches": 613, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1163, "tokens": 94702057, "loss_tokens": 88028, "loss": 2.6500078455150633, "lr": 0.000299850124867567, "optimizer_lrs": [0.01999000832450447, 0.000299850124867567], "gradient_norm": 0.5963953733444214, "seconds": 0.32411702899997863, "nonpadding_tokens_per_second": 271593.2583721351, "ranks": [{"compute_seconds": 0.32100062799997886, "nonpadding_tokens": 16901, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32142299200000934, "nonpadding_tokens": 11587, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31852936399997134, "nonpadding_tokens": 3626, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31983676200002265, "nonpadding_tokens": 16600, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3212534049999931, "nonpadding_tokens": 4015, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3203363609999883, "nonpadding_tokens": 4248, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31730242499997985, "nonpadding_tokens": 14474, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32179958200003966, "nonpadding_tokens": 16577, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004214129999695615, "prefetch": {"data_wait_seconds": 0.005809764999597178, "consumed_batches": 614, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1164, "tokens": 94765888, "loss_tokens": 63831, "loss": 2.7642289796493866, "lr": 0.0002998498686766964, "optimizer_lrs": [0.019989991245113095, 0.0002998498686766964], "gradient_norm": 0.42364516854286194, "seconds": 0.2371900329999903, "nonpadding_tokens_per_second": 269113.33158759924, "ranks": [{"compute_seconds": 0.2326123930000108, "nonpadding_tokens": 7950, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23479834500000152, "nonpadding_tokens": 12957, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2339479640000377, "nonpadding_tokens": 23585, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23424339299998564, "nonpadding_tokens": 3967, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23071693800000048, "nonpadding_tokens": 6127, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23193324899995105, "nonpadding_tokens": 4434, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23088991700001316, "nonpadding_tokens": 1992, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2316093789999627, "nonpadding_tokens": 2819, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038728000004084606, "prefetch": {"data_wait_seconds": 0.0058177449996179575, "consumed_batches": 615, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1165, "tokens": 94825465, "loss_tokens": 59577, "loss": 2.565447603521493, "lr": 0.000299849629362318, "optimizer_lrs": [0.0199899752908212, 0.000299849629362318], "gradient_norm": 0.45956382155418396, "seconds": 0.2794286459999853, "nonpadding_tokens_per_second": 213210.06579977894, "ranks": [{"compute_seconds": 0.2749268889999712, "nonpadding_tokens": 2332, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27302943900002674, "nonpadding_tokens": 4077, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2736046620000252, "nonpadding_tokens": 6841, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2746316600000114, "nonpadding_tokens": 11557, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2726735420000068, "nonpadding_tokens": 14200, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27394208300000855, "nonpadding_tokens": 6765, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2726824610000449, "nonpadding_tokens": 8145, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27708105199997135, "nonpadding_tokens": 5660, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039364600002045336, "prefetch": {"data_wait_seconds": 0.005825847999645362, "consumed_batches": 616, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1166, "tokens": 94894798, "loss_tokens": 69333, "loss": 2.949861087072534, "lr": 0.0002998493506193194, "optimizer_lrs": [0.01998995670795463, 0.0002998493506193194], "gradient_norm": 0.4295797348022461, "seconds": 0.3216511999999625, "nonpadding_tokens_per_second": 215553.3696128231, "ranks": [{"compute_seconds": 0.31513718799999424, "nonpadding_tokens": 9004, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3148199920000252, "nonpadding_tokens": 4854, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3173719399999868, "nonpadding_tokens": 4581, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.318389959000001, "nonpadding_tokens": 5894, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3191236949999734, "nonpadding_tokens": 7382, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31552556600001935, "nonpadding_tokens": 11395, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31805771099999447, "nonpadding_tokens": 10031, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3191416679999861, "nonpadding_tokens": 16192, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033938699999680466, "prefetch": {"data_wait_seconds": 0.005833747999659522, "consumed_batches": 617, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1167, "tokens": 94955148, "loss_tokens": 60350, "loss": 2.9230659693454846, "lr": 0.0002998491077811337, "optimizer_lrs": [0.019989940518742248, 0.0002998491077811337], "gradient_norm": 0.4558244049549103, "seconds": 0.2441535730000055, "nonpadding_tokens_per_second": 247180.49078068841, "ranks": [{"compute_seconds": 0.239524222, "nonpadding_tokens": 7112, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24179618800002345, "nonpadding_tokens": 6607, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24055205400003388, "nonpadding_tokens": 4783, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24108260899998868, "nonpadding_tokens": 3501, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24149606400004586, "nonpadding_tokens": 5812, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2387524449999887, "nonpadding_tokens": 12457, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24114682499998707, "nonpadding_tokens": 15257, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23858767600000874, "nonpadding_tokens": 4821, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039080599998442267, "prefetch": {"data_wait_seconds": 0.00584358099963822, "consumed_batches": 618, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1168, "tokens": 94996669, "loss_tokens": 41521, "loss": 2.843252887093278, "lr": 0.0002998489405941621, "optimizer_lrs": [0.01998992937294414, 0.0002998489405941621], "gradient_norm": 0.49437758326530457, "seconds": 0.2261232609999979, "nonpadding_tokens_per_second": 183621.0915072571, "ranks": [{"compute_seconds": 0.22298531299998103, "nonpadding_tokens": 1434, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22180770399995708, "nonpadding_tokens": 11541, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21912802100001727, "nonpadding_tokens": 5653, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21966699300003256, "nonpadding_tokens": 2812, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.22291217700001198, "nonpadding_tokens": 7069, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2209320649999995, "nonpadding_tokens": 5078, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2227367499999673, "nonpadding_tokens": 5155, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.223795999999993, "nonpadding_tokens": 2779, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040339299999914147, "prefetch": {"data_wait_seconds": 0.005852444999632667, "consumed_batches": 619, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1169, "tokens": 95067453, "loss_tokens": 70784, "loss": 2.7079144386443263, "lr": 0.00029984865536454157, "optimizer_lrs": [0.019989910357636108, 0.00029984865536454157], "gradient_norm": 0.44019395112991333, "seconds": 0.2763714719999939, "nonpadding_tokens_per_second": 256119.05414029694, "ranks": [{"compute_seconds": 0.2735182779999832, "nonpadding_tokens": 2418, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2731507640000359, "nonpadding_tokens": 15863, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2699871190000067, "nonpadding_tokens": 4848, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27095432400000163, "nonpadding_tokens": 2084, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27310510199998816, "nonpadding_tokens": 22408, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2702099229999817, "nonpadding_tokens": 11433, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27272216199997956, "nonpadding_tokens": 5556, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27408438700001625, "nonpadding_tokens": 6174, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003579699999818331, "prefetch": {"data_wait_seconds": 0.00586193899961529, "consumed_batches": 620, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1170, "tokens": 95121573, "loss_tokens": 54120, "loss": 2.8742277000184773, "lr": 0.00029984843710242106, "optimizer_lrs": [0.019989895806828074, 0.00029984843710242106], "gradient_norm": 0.46940383315086365, "seconds": 0.2777432369999815, "nonpadding_tokens_per_second": 194856.2297486423, "ranks": [{"compute_seconds": 0.275380226999971, "nonpadding_tokens": 3742, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.274902700000041, "nonpadding_tokens": 4756, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27222181100000853, "nonpadding_tokens": 2635, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2728185919999646, "nonpadding_tokens": 6590, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27514311499999167, "nonpadding_tokens": 12313, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27410873700000593, "nonpadding_tokens": 6462, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2751153380000346, "nonpadding_tokens": 7784, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2754149879999659, "nonpadding_tokens": 9838, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003763909999747739, "prefetch": {"data_wait_seconds": 0.005871396999637568, "consumed_batches": 621, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1171, "tokens": 95180028, "loss_tokens": 58455, "loss": 2.5071123086134635, "lr": 0.0002998482011809816, "optimizer_lrs": [0.01998988007873211, 0.0002998482011809816], "gradient_norm": 0.39524540305137634, "seconds": 0.3175227979999704, "nonpadding_tokens_per_second": 184097.01718490603, "ranks": [{"compute_seconds": 0.31509526999997206, "nonpadding_tokens": 13653, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31440784400001576, "nonpadding_tokens": 15944, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31158002299997634, "nonpadding_tokens": 4052, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31186370099999294, "nonpadding_tokens": 7251, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3144095870000001, "nonpadding_tokens": 2897, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3132442040000001, "nonpadding_tokens": 3740, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3141491540000061, "nonpadding_tokens": 2573, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3114593229999514, "nonpadding_tokens": 8345, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042244799999480165, "prefetch": {"data_wait_seconds": 0.005880335999634667, "consumed_batches": 622, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1172, "tokens": 95227650, "loss_tokens": 47622, "loss": 2.5667230350468273, "lr": 0.00029984800884543975, "optimizer_lrs": [0.01998986725636265, 0.00029984800884543975], "gradient_norm": 0.49144792556762695, "seconds": 0.4804446819999839, "nonpadding_tokens_per_second": 99120.6725439446, "ranks": [{"compute_seconds": 0.47702029200002016, "nonpadding_tokens": 7280, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.47755562600002577, "nonpadding_tokens": 2088, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.47606635699997923, "nonpadding_tokens": 3607, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.47486297599999716, "nonpadding_tokens": 2685, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.47720635499996433, "nonpadding_tokens": 3257, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4743272910000087, "nonpadding_tokens": 5873, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.4770856260000187, "nonpadding_tokens": 4927, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.47805782900002214, "nonpadding_tokens": 17905, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003962010000009286, "prefetch": {"data_wait_seconds": 0.005888739999605264, "consumed_batches": 623, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1173, "tokens": 95296037, "loss_tokens": 68387, "loss": 3.09909069889014, "lr": 0.00029984773243142633, "optimizer_lrs": [0.019989848828761758, 0.00029984773243142633], "gradient_norm": 1.4591329097747803, "seconds": 0.19499563599998737, "nonpadding_tokens_per_second": 350710.41282177425, "ranks": [{"compute_seconds": 0.19241059799998084, "nonpadding_tokens": 2104, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19278085999997074, "nonpadding_tokens": 5627, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19168463900001598, "nonpadding_tokens": 8358, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.1922948120000001, "nonpadding_tokens": 10515, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.19257528700001103, "nonpadding_tokens": 6640, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.18987326600000642, "nonpadding_tokens": 5661, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.1906944500000236, "nonpadding_tokens": 15806, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.1894350050000071, "nonpadding_tokens": 13676, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003755530000262297, "prefetch": {"data_wait_seconds": 0.0058982529996143285, "consumed_batches": 624, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1174, "tokens": 95358938, "loss_tokens": 62901, "loss": 2.6015728088583647, "lr": 0.00029984747796974094, "optimizer_lrs": [0.019989831864649397, 0.00029984747796974094], "gradient_norm": 1.389218807220459, "seconds": 0.4127527830000304, "nonpadding_tokens_per_second": 152393.88464643102, "ranks": [{"compute_seconds": 0.41023967699999275, "nonpadding_tokens": 4045, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4096751989999916, "nonpadding_tokens": 10337, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4067564690000154, "nonpadding_tokens": 10220, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4090774879999799, "nonpadding_tokens": 10385, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4097779360000118, "nonpadding_tokens": 4350, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4085402050000084, "nonpadding_tokens": 8631, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.4094171500000243, "nonpadding_tokens": 2292, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4103053169999953, "nonpadding_tokens": 12641, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003668230000357653, "prefetch": {"data_wait_seconds": 0.0059067469996563204, "consumed_batches": 625, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1175, "tokens": 95414026, "loss_tokens": 55088, "loss": 2.898245477690241, "lr": 0.0002998472549406092, "optimizer_lrs": [0.019989816996040615, 0.0002998472549406092], "gradient_norm": 0.44600245356559753, "seconds": 0.1993983009999738, "nonpadding_tokens_per_second": 276271.1604047581, "ranks": [{"compute_seconds": 0.19688446200001408, "nonpadding_tokens": 2598, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19293702999999596, "nonpadding_tokens": 5001, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19569670900000347, "nonpadding_tokens": 14260, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.19431093999997984, "nonpadding_tokens": 4091, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.19686462399999982, "nonpadding_tokens": 9015, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19424671799998805, "nonpadding_tokens": 10053, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.19254084599998578, "nonpadding_tokens": 7830, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.1929094920000125, "nonpadding_tokens": 2240, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039290099999789163, "prefetch": {"data_wait_seconds": 0.005915131999657319, "consumed_batches": 626, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1176, "tokens": 95487282, "loss_tokens": 73256, "loss": 2.5109956863601615, "lr": 0.0002998469581042869, "optimizer_lrs": [0.019989797206952462, 0.0002998469581042869], "gradient_norm": 0.9258591532707214, "seconds": 0.22420257099997798, "nonpadding_tokens_per_second": 326740.2317166434, "ranks": [{"compute_seconds": 0.22140124899999591, "nonpadding_tokens": 2581, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21808546300002263, "nonpadding_tokens": 16831, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22116032600001745, "nonpadding_tokens": 17572, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2216904340000383, "nonpadding_tokens": 2435, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.22194455500005006, "nonpadding_tokens": 4840, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2190523639999924, "nonpadding_tokens": 5416, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.22195516900001167, "nonpadding_tokens": 14835, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2186900430000378, "nonpadding_tokens": 8746, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003502330000060283, "prefetch": {"data_wait_seconds": 0.0059240489996597034, "consumed_batches": 627, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1177, "tokens": 95526486, "loss_tokens": 39204, "loss": 2.794851928374656, "lr": 0.00029984679912976897, "optimizer_lrs": [0.019989786608651267, 0.00029984679912976897], "gradient_norm": 0.4327623248100281, "seconds": 0.22964925200000152, "nonpadding_tokens_per_second": 170712.5090048181, "ranks": [{"compute_seconds": 0.22666575300002023, "nonpadding_tokens": 4365, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22735491700001376, "nonpadding_tokens": 4368, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22422533400003886, "nonpadding_tokens": 2181, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22655929500001548, "nonpadding_tokens": 5387, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2228131850000068, "nonpadding_tokens": 1299, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22399922899995772, "nonpadding_tokens": 7310, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2230055640000046, "nonpadding_tokens": 9608, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2273016439999651, "nonpadding_tokens": 4686, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038250599999400947, "prefetch": {"data_wait_seconds": 0.005932500999691115, "consumed_batches": 628, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1178, "tokens": 95608802, "loss_tokens": 82316, "loss": 2.857666834212061, "lr": 0.0002998464650651634, "optimizer_lrs": [0.01998976433767756, 0.0002998464650651634], "gradient_norm": 0.511963427066803, "seconds": 0.24845058900001504, "nonpadding_tokens_per_second": 331317.38721696095, "ranks": [{"compute_seconds": 0.2462181649999593, "nonpadding_tokens": 2847, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24191720499999292, "nonpadding_tokens": 4884, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2445437359999687, "nonpadding_tokens": 4901, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24508823899998333, "nonpadding_tokens": 5360, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24140373500000578, "nonpadding_tokens": 15850, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24449585800005025, "nonpadding_tokens": 10709, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24542601500002093, "nonpadding_tokens": 18363, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2421725669999546, "nonpadding_tokens": 19402, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034733000001097025, "prefetch": {"data_wait_seconds": 0.005940732999661691, "consumed_batches": 629, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1179, "tokens": 95787877, "loss_tokens": 179075, "loss": 3.124676113360324, "lr": 0.0002998457370656877, "optimizer_lrs": [0.019989715804379183, 0.0002998457370656877], "gradient_norm": 0.4794475734233856, "seconds": 0.3780377040000076, "nonpadding_tokens_per_second": 473696.1369334642, "ranks": [{"compute_seconds": 0.3748614959999941, "nonpadding_tokens": 24883, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3756345749999923, "nonpadding_tokens": 32130, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.374624348999987, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.37503976200002853, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3757040429999847, "nonpadding_tokens": 30185, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37425603699995236, "nonpadding_tokens": 14340, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3717562310000062, "nonpadding_tokens": 6046, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3719887569999969, "nonpadding_tokens": 5955, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003997740000158956, "prefetch": {"data_wait_seconds": 0.005949649999664075, "consumed_batches": 630, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1180, "tokens": 95828367, "loss_tokens": 40490, "loss": 2.7297125833539146, "lr": 0.00029984557222182236, "optimizer_lrs": [0.019989704814788158, 0.00029984557222182236], "gradient_norm": 0.48481014370918274, "seconds": 0.26741624599998204, "nonpadding_tokens_per_second": 151411.8929034802, "ranks": [{"compute_seconds": 0.2650981869999782, "nonpadding_tokens": 10484, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26253899699997874, "nonpadding_tokens": 3095, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26291910899999493, "nonpadding_tokens": 5544, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26391000299997813, "nonpadding_tokens": 7567, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2621412990000067, "nonpadding_tokens": 2879, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.263179230999981, "nonpadding_tokens": 1986, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26168422599999985, "nonpadding_tokens": 5940, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2630860809999831, "nonpadding_tokens": 2995, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038319999998748244, "prefetch": {"data_wait_seconds": 0.005959398999678456, "consumed_batches": 631, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1181, "tokens": 95874767, "loss_tokens": 46400, "loss": 2.6920858028017243, "lr": 0.00029984538320884527, "optimizer_lrs": [0.019989692213923018, 0.00029984538320884527], "gradient_norm": 0.4725208580493927, "seconds": 0.24083284400001048, "nonpadding_tokens_per_second": 192664.7513243583, "ranks": [{"compute_seconds": 0.23682715800003962, "nonpadding_tokens": 3205, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23543309199999385, "nonpadding_tokens": 16859, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2361579579999784, "nonpadding_tokens": 1524, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2384440980000022, "nonpadding_tokens": 6990, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23470789199996034, "nonpadding_tokens": 2457, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2375707519999537, "nonpadding_tokens": 3569, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23495880699999816, "nonpadding_tokens": 4315, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23588708299996597, "nonpadding_tokens": 7481, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041168399997104643, "prefetch": {"data_wait_seconds": 0.0059692489996479026, "consumed_batches": 632, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1182, "tokens": 95924370, "loss_tokens": 49603, "loss": 2.734063463903393, "lr": 0.0002998451810204965, "optimizer_lrs": [0.01998967873469977, 0.0002998451810204965], "gradient_norm": 0.4539915919303894, "seconds": 0.2742817409999816, "nonpadding_tokens_per_second": 180846.89057009935, "ranks": [{"compute_seconds": 0.27203851299998405, "nonpadding_tokens": 5837, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27118988099999797, "nonpadding_tokens": 4266, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27067651500004786, "nonpadding_tokens": 14562, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26889764000003424, "nonpadding_tokens": 6710, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27128666999999496, "nonpadding_tokens": 5055, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27018112799999017, "nonpadding_tokens": 2235, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27119337700003143, "nonpadding_tokens": 8515, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27186934500002735, "nonpadding_tokens": 2423, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039219699999648583, "prefetch": {"data_wait_seconds": 0.005994375999662793, "consumed_batches": 633, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1183, "tokens": 96008550, "loss_tokens": 84180, "loss": 2.8653200359349014, "lr": 0.00029984483758957697, "optimizer_lrs": [0.019989655839305134, 0.00029984483758957697], "gradient_norm": 0.734259307384491, "seconds": 0.36804574500001763, "nonpadding_tokens_per_second": 228721.56829308262, "ranks": [{"compute_seconds": 0.3643186410000112, "nonpadding_tokens": 9336, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3628930390000278, "nonpadding_tokens": 18511, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3654983819999984, "nonpadding_tokens": 4305, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3640187049999781, "nonpadding_tokens": 12108, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.36298141500003567, "nonpadding_tokens": 9899, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36557456799999954, "nonpadding_tokens": 2105, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.36234190400000443, "nonpadding_tokens": 2554, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36312663500001463, "nonpadding_tokens": 25362, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005624130000114747, "prefetch": {"data_wait_seconds": 0.0060019359996914545, "consumed_batches": 634, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1184, "tokens": 96079160, "loss_tokens": 70610, "loss": 2.7197820776094037, "lr": 0.00029984454922721114, "optimizer_lrs": [0.01998963661514741, 0.00029984454922721114], "gradient_norm": 0.4255819618701935, "seconds": 0.34413279500000726, "nonpadding_tokens_per_second": 205182.4209314271, "ranks": [{"compute_seconds": 0.3417598869999665, "nonpadding_tokens": 10341, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34169440000005125, "nonpadding_tokens": 6922, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33872692800002824, "nonpadding_tokens": 1771, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3414669379999964, "nonpadding_tokens": 9859, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33770962500000223, "nonpadding_tokens": 12012, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34036075400001664, "nonpadding_tokens": 2763, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.34175275100000135, "nonpadding_tokens": 21645, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3381689789999882, "nonpadding_tokens": 5297, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004053119999980481, "prefetch": {"data_wait_seconds": 0.006010691999691176, "consumed_batches": 635, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1185, "tokens": 96129599, "loss_tokens": 50439, "loss": 2.9981831395348837, "lr": 0.00029984434307685226, "optimizer_lrs": [0.019989622871790153, 0.00029984434307685226], "gradient_norm": 0.4324398636817932, "seconds": 0.3297830810000164, "nonpadding_tokens_per_second": 152945.99057978202, "ranks": [{"compute_seconds": 0.3247838609999576, "nonpadding_tokens": 5238, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32391185599999517, "nonpadding_tokens": 14961, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3239767300000267, "nonpadding_tokens": 5398, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3253015340000047, "nonpadding_tokens": 2542, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32736247299999377, "nonpadding_tokens": 4698, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3264626030000386, "nonpadding_tokens": 11357, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3233422889999815, "nonpadding_tokens": 1982, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.323885654000037, "nonpadding_tokens": 4263, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040531799999143914, "prefetch": {"data_wait_seconds": 0.006020779999701631, "consumed_batches": 636, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1186, "tokens": 96191485, "loss_tokens": 61886, "loss": 2.960754199253466, "lr": 0.00029984408995469057, "optimizer_lrs": [0.01998960599697937, 0.00029984408995469057], "gradient_norm": 0.4348396360874176, "seconds": 0.23330440400002317, "nonpadding_tokens_per_second": 265258.60180502146, "ranks": [{"compute_seconds": 0.2303959689999715, "nonpadding_tokens": 7863, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23024117699998214, "nonpadding_tokens": 1316, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22734849600004736, "nonpadding_tokens": 7744, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22825716800002738, "nonpadding_tokens": 3177, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.22642089199996462, "nonpadding_tokens": 9512, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22939321500001597, "nonpadding_tokens": 17062, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23027859900003023, "nonpadding_tokens": 12383, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2310060579999913, "nonpadding_tokens": 2829, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036309099999698446, "prefetch": {"data_wait_seconds": 0.006029477999746291, "consumed_batches": 637, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1187, "tokens": 96298263, "loss_tokens": 106778, "loss": 2.789656167468954, "lr": 0.00029984365273491744, "optimizer_lrs": [0.019989576848994498, 0.00029984365273491744], "gradient_norm": 0.43440258502960205, "seconds": 0.27838698499999737, "nonpadding_tokens_per_second": 383559.5978023219, "ranks": [{"compute_seconds": 0.2756305730000008, "nonpadding_tokens": 4476, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2721697100000142, "nonpadding_tokens": 4502, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2729718169999842, "nonpadding_tokens": 18603, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2737835190000055, "nonpadding_tokens": 9735, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2760992629999919, "nonpadding_tokens": 19908, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2749142790000292, "nonpadding_tokens": 3835, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2722974289999911, "nonpadding_tokens": 19973, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27279765100001896, "nonpadding_tokens": 25746, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035970900000847905, "prefetch": {"data_wait_seconds": 0.006039564999767322, "consumed_batches": 638, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1188, "tokens": 96364811, "loss_tokens": 66548, "loss": 2.7819102377231473, "lr": 0.0002998433799338937, "optimizer_lrs": [0.01998955866225958, 0.0002998433799338937], "gradient_norm": 0.8629067540168762, "seconds": 0.273140791000003, "nonpadding_tokens_per_second": 243639.91828668048, "ranks": [{"compute_seconds": 0.2689006530000029, "nonpadding_tokens": 7920, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2672615679999808, "nonpadding_tokens": 2324, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2702487600000154, "nonpadding_tokens": 6904, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26828221400000984, "nonpadding_tokens": 13821, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2708487780000155, "nonpadding_tokens": 17669, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2700566899999899, "nonpadding_tokens": 3453, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.270818711000004, "nonpadding_tokens": 5212, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2676414509999745, "nonpadding_tokens": 9245, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034553799997638635, "prefetch": {"data_wait_seconds": 0.006048746999752552, "consumed_batches": 639, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1189, "tokens": 96413502, "loss_tokens": 48691, "loss": 3.071635736070321, "lr": 0.0002998431801837338, "optimizer_lrs": [0.019989545345582255, 0.0002998431801837338], "gradient_norm": 0.5202891230583191, "seconds": 0.2858777960000225, "nonpadding_tokens_per_second": 170321.02766035096, "ranks": [{"compute_seconds": 0.28343907499998977, "nonpadding_tokens": 2848, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28363253899999563, "nonpadding_tokens": 1874, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28063905200002637, "nonpadding_tokens": 14408, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28102895099999614, "nonpadding_tokens": 12103, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27981410299997833, "nonpadding_tokens": 4036, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28144913899996027, "nonpadding_tokens": 2685, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28359719399998085, "nonpadding_tokens": 1763, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2798130609999703, "nonpadding_tokens": 8974, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034144700003935213, "prefetch": {"data_wait_seconds": 0.006057186999740338, "consumed_batches": 640, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1190, "tokens": 96457067, "loss_tokens": 43565, "loss": 3.1591174824974178, "lr": 0.00029984300135465944, "optimizer_lrs": [0.019989533423643964, 0.00029984300135465944], "gradient_norm": 0.5098366141319275, "seconds": 0.2829042770000001, "nonpadding_tokens_per_second": 153992.0161758459, "ranks": [{"compute_seconds": 0.2800686859999928, "nonpadding_tokens": 7584, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27678227000001243, "nonpadding_tokens": 3733, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2772980880000091, "nonpadding_tokens": 4116, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27780740899999046, "nonpadding_tokens": 3392, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2759515469999769, "nonpadding_tokens": 1475, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27870577599998114, "nonpadding_tokens": 15018, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2761453269999947, "nonpadding_tokens": 3228, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28057463199996846, "nonpadding_tokens": 5019, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003612330000350994, "prefetch": {"data_wait_seconds": 0.006065229999720145, "consumed_batches": 641, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1191, "tokens": 96556965, "loss_tokens": 99898, "loss": 2.7632363135398106, "lr": 0.0002998425909008744, "optimizer_lrs": [0.019989506060058292, 0.0002998425909008744], "gradient_norm": 0.48952460289001465, "seconds": 0.31177790700002106, "nonpadding_tokens_per_second": 320413.9798141414, "ranks": [{"compute_seconds": 0.3051586999999927, "nonpadding_tokens": 11546, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30629853299996057, "nonpadding_tokens": 11919, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3080102720000468, "nonpadding_tokens": 9263, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3089829239999631, "nonpadding_tokens": 3618, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30944920199999615, "nonpadding_tokens": 7044, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3050718759999995, "nonpadding_tokens": 13555, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.30212212600002886, "nonpadding_tokens": 17301, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30277504700001145, "nonpadding_tokens": 25652, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037736399997356784, "prefetch": {"data_wait_seconds": 0.006073830999710594, "consumed_batches": 642, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1192, "tokens": 96618559, "loss_tokens": 61594, "loss": 2.7944272676721758, "lr": 0.0002998423375609484, "optimizer_lrs": [0.019989489170729895, 0.0002998423375609484], "gradient_norm": 0.45728904008865356, "seconds": 0.3907834199999911, "nonpadding_tokens_per_second": 157616.7177205251, "ranks": [{"compute_seconds": 0.3847571140000241, "nonpadding_tokens": 4732, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3865656890000082, "nonpadding_tokens": 12644, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3863083220000476, "nonpadding_tokens": 11157, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.38831262099995456, "nonpadding_tokens": 3284, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3876962740000067, "nonpadding_tokens": 6819, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38520299599997543, "nonpadding_tokens": 1924, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3823281580000071, "nonpadding_tokens": 15869, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.38834958799998276, "nonpadding_tokens": 5165, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003984789999549321, "prefetch": {"data_wait_seconds": 0.0060849179997291, "consumed_batches": 643, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1193, "tokens": 96687243, "loss_tokens": 68684, "loss": 3.0082752169355307, "lr": 0.00029984205481933004, "optimizer_lrs": [0.01998947032128867, 0.00029984205481933004], "gradient_norm": 0.5102543830871582, "seconds": 0.36021072800002685, "nonpadding_tokens_per_second": 190677.27488670155, "ranks": [{"compute_seconds": 0.3540675570000076, "nonpadding_tokens": 10678, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35761375999999245, "nonpadding_tokens": 24790, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3546571670000276, "nonpadding_tokens": 4280, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3552893859999813, "nonpadding_tokens": 5854, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3578246889999832, "nonpadding_tokens": 1652, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3566057479999927, "nonpadding_tokens": 8192, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.35355566399999816, "nonpadding_tokens": 4197, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35447601300000997, "nonpadding_tokens": 9041, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003368759999489157, "prefetch": {"data_wait_seconds": 0.006093258999683826, "consumed_batches": 644, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1194, "tokens": 96751452, "loss_tokens": 64209, "loss": 2.838470852995686, "lr": 0.00029984179027035867, "optimizer_lrs": [0.01998945268469058, 0.00029984179027035867], "gradient_norm": 0.5739414095878601, "seconds": 0.24896022100000437, "nonpadding_tokens_per_second": 257908.6720846013, "ranks": [{"compute_seconds": 0.24512085300000308, "nonpadding_tokens": 4155, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2458148469999628, "nonpadding_tokens": 3981, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24538390499998286, "nonpadding_tokens": 10589, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.246207217999995, "nonpadding_tokens": 4690, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2464779669999757, "nonpadding_tokens": 10731, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24462586499998906, "nonpadding_tokens": 6743, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24200288300005468, "nonpadding_tokens": 10979, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24230935900004624, "nonpadding_tokens": 12341, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044119199998249314, "prefetch": {"data_wait_seconds": 0.006101422999691408, "consumed_batches": 645, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1195, "tokens": 96794843, "loss_tokens": 43391, "loss": 2.879984112488765, "lr": 0.00029984161136883573, "optimizer_lrs": [0.019989440757922382, 0.00029984161136883573], "gradient_norm": 0.48368608951568604, "seconds": 0.32674392300003774, "nonpadding_tokens_per_second": 132798.18520142758, "ranks": [{"compute_seconds": 0.3199056870000163, "nonpadding_tokens": 4063, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3191361830000119, "nonpadding_tokens": 4678, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3192454520000183, "nonpadding_tokens": 9844, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32034040100000993, "nonpadding_tokens": 4126, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31829433400002927, "nonpadding_tokens": 9820, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3214290659999506, "nonpadding_tokens": 2507, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.32306211400003804, "nonpadding_tokens": 5962, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3242084400000067, "nonpadding_tokens": 2391, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004095269999879747, "prefetch": {"data_wait_seconds": 0.006110338999704368, "consumed_batches": 646, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1196, "tokens": 96894386, "loss_tokens": 99543, "loss": 2.9607945812362497, "lr": 0.00029984120057030864, "optimizer_lrs": [0.01998941337135391, 0.00029984120057030864], "gradient_norm": 0.5606759190559387, "seconds": 0.3430578260000061, "nonpadding_tokens_per_second": 290163.9095678238, "ranks": [{"compute_seconds": 0.34069512800004986, "nonpadding_tokens": 24911, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3403551960000186, "nonpadding_tokens": 14895, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33756073299997524, "nonpadding_tokens": 15195, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34029594499997984, "nonpadding_tokens": 5016, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34050701999996136, "nonpadding_tokens": 11406, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3390314379999495, "nonpadding_tokens": 12565, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.33658133200003704, "nonpadding_tokens": 4464, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3374192220000509, "nonpadding_tokens": 11091, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003773409999894284, "prefetch": {"data_wait_seconds": 0.006119301999731874, "consumed_batches": 647, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1197, "tokens": 96967517, "loss_tokens": 73131, "loss": 2.631924645499173, "lr": 0.0002998408984311957, "optimizer_lrs": [0.01998939322874638, 0.0002998408984311957], "gradient_norm": 0.3834141194820404, "seconds": 0.27130170199995973, "nonpadding_tokens_per_second": 269555.99416037154, "ranks": [{"compute_seconds": 0.26891198999999233, "nonpadding_tokens": 5317, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26698806299998523, "nonpadding_tokens": 11906, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26726141599999664, "nonpadding_tokens": 4300, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2682524309999508, "nonpadding_tokens": 12383, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2668031229999883, "nonpadding_tokens": 4722, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26894361799998023, "nonpadding_tokens": 9969, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26635475399996267, "nonpadding_tokens": 13966, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2668060049999781, "nonpadding_tokens": 10568, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035050299999284107, "prefetch": {"data_wait_seconds": 0.0061282619997342636, "consumed_batches": 648, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1198, "tokens": 97045598, "loss_tokens": 78081, "loss": 2.9704877787169734, "lr": 0.00029984057552448224, "optimizer_lrs": [0.01998937170163215, 0.00029984057552448224], "gradient_norm": 0.6447332501411438, "seconds": 0.295680888999982, "nonpadding_tokens_per_second": 264071.85213787947, "ranks": [{"compute_seconds": 0.29227687100001276, "nonpadding_tokens": 19687, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2894861539999738, "nonpadding_tokens": 14572, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2898573510000233, "nonpadding_tokens": 5663, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2904112339999756, "nonpadding_tokens": 6192, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2934384040000282, "nonpadding_tokens": 5338, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29097960699999703, "nonpadding_tokens": 8204, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2889896730000032, "nonpadding_tokens": 5461, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28986191399997097, "nonpadding_tokens": 12964, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000364326000010351, "prefetch": {"data_wait_seconds": 0.006136518999724672, "consumed_batches": 649, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1199, "tokens": 97134011, "loss_tokens": 88413, "loss": 2.7404918889190504, "lr": 0.00029984020949446664, "optimizer_lrs": [0.019989347299631113, 0.00029984020949446664], "gradient_norm": 0.4884069859981537, "seconds": 0.28403021700000863, "nonpadding_tokens_per_second": 311280.26071957446, "ranks": [{"compute_seconds": 0.28132807000002913, "nonpadding_tokens": 6693, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28142427900002076, "nonpadding_tokens": 6038, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27805546199999753, "nonpadding_tokens": 8851, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2790670850000083, "nonpadding_tokens": 11460, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28122365000001537, "nonpadding_tokens": 10031, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2787750300000198, "nonpadding_tokens": 29200, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27704857899999524, "nonpadding_tokens": 9067, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2817342469999744, "nonpadding_tokens": 7073, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003903299999592491, "prefetch": {"data_wait_seconds": 0.006145956999731084, "consumed_batches": 650, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1200, "tokens": 97203079, "loss_tokens": 69068, "loss": 2.945341230743615, "lr": 0.00029983992326096, "optimizer_lrs": [0.019989328217397337, 0.00029983992326096], "gradient_norm": 0.4757292866706848, "seconds": 0.3383167940000362, "nonpadding_tokens_per_second": 204151.851829125, "ranks": [{"compute_seconds": 0.333085374999996, "nonpadding_tokens": 20133, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33494664299996657, "nonpadding_tokens": 12219, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3323079639999946, "nonpadding_tokens": 2630, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33512610299999324, "nonpadding_tokens": 4350, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33143826200000603, "nonpadding_tokens": 5366, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3327106840000056, "nonpadding_tokens": 5317, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3349156940000171, "nonpadding_tokens": 12995, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3359147099999973, "nonpadding_tokens": 6058, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003947449999941455, "prefetch": {"data_wait_seconds": 0.006155225999705749, "consumed_batches": 651, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1201, "tokens": 97258798, "loss_tokens": 55719, "loss": 2.7564671611120084, "lr": 0.0002998396921621936, "optimizer_lrs": [0.01998931281081291, 0.0002998396921621936], "gradient_norm": 0.410453736782074, "seconds": 0.3337549240000044, "nonpadding_tokens_per_second": 166945.85156142674, "ranks": [{"compute_seconds": 0.23016292200003363, "nonpadding_tokens": 5867, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29625112899998385, "nonpadding_tokens": 5823, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2955718809999439, "nonpadding_tokens": 9049, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2939504839999927, "nonpadding_tokens": 8917, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29645810800002437, "nonpadding_tokens": 2994, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2939150930000096, "nonpadding_tokens": 6397, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.29648333300002605, "nonpadding_tokens": 3471, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29302934000003233, "nonpadding_tokens": 13201, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.025835337999978947, "prefetch": {"data_wait_seconds": 0.006164248999724009, "consumed_batches": 652, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1202, "tokens": 97335021, "loss_tokens": 76223, "loss": 2.9023584580769586, "lr": 0.0002998393757517018, "optimizer_lrs": [0.019989291716780124, 0.0002998393757517018], "gradient_norm": 0.4943777918815613, "seconds": 0.24833800900000824, "nonpadding_tokens_per_second": 306932.4760512092, "ranks": [{"compute_seconds": 0.24341698399996403, "nonpadding_tokens": 3943, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24544715100000758, "nonpadding_tokens": 2407, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2425127669999938, "nonpadding_tokens": 12594, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24433450199995832, "nonpadding_tokens": 12848, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24277048200002582, "nonpadding_tokens": 8580, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24413985999996157, "nonpadding_tokens": 16241, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24151338799998712, "nonpadding_tokens": 8625, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24597973599998113, "nonpadding_tokens": 10985, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003402730000061638, "prefetch": {"data_wait_seconds": 0.006172691999736912, "consumed_batches": 653, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1203, "tokens": 97397042, "loss_tokens": 62021, "loss": 2.7860850558681736, "lr": 0.00029983911806531007, "optimizer_lrs": [0.01998927453768734, 0.00029983911806531007], "gradient_norm": 0.6088539958000183, "seconds": 0.314479480999978, "nonpadding_tokens_per_second": 197217.9545793779, "ranks": [{"compute_seconds": 0.30998555899998337, "nonpadding_tokens": 11363, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3090193259999978, "nonpadding_tokens": 3662, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30917367800003603, "nonpadding_tokens": 10927, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3097392409999884, "nonpadding_tokens": 3935, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3079106249999768, "nonpadding_tokens": 6776, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3108845730000098, "nonpadding_tokens": 16052, "peak_allocated_bytes": 14991212032, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3080277019999471, "nonpadding_tokens": 4869, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31216973400000825, "nonpadding_tokens": 4437, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035943099999258266, "prefetch": {"data_wait_seconds": 0.0061813669997405896, "consumed_batches": 654, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1204, "tokens": 97461177, "loss_tokens": 64135, "loss": 3.1321438664535743, "lr": 0.0002998388513785335, "optimizer_lrs": [0.019989256758568903, 0.0002998388513785335], "gradient_norm": 0.453319787979126, "seconds": 0.3285481330000266, "nonpadding_tokens_per_second": 195207.31837485378, "ranks": [{"compute_seconds": 0.3260140919999799, "nonpadding_tokens": 4202, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32244781899999, "nonpadding_tokens": 6062, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32263147500003697, "nonpadding_tokens": 8654, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3233179730000302, "nonpadding_tokens": 10067, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32166775700000017, "nonpadding_tokens": 9516, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32469667500004107, "nonpadding_tokens": 15286, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.32195097900000746, "nonpadding_tokens": 5677, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32619834400003356, "nonpadding_tokens": 4671, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004113100000040504, "prefetch": {"data_wait_seconds": 0.006188692999742784, "consumed_batches": 655, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1205, "tokens": 97576099, "loss_tokens": 114922, "loss": 2.710129479124972, "lr": 0.0002998383729566597, "optimizer_lrs": [0.019989224863777317, 0.0002998383729566597], "gradient_norm": 0.6655506491661072, "seconds": 0.33446365500003594, "nonpadding_tokens_per_second": 343600.8615046309, "ranks": [{"compute_seconds": 0.33140019600000414, "nonpadding_tokens": 16983, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32884225800000877, "nonpadding_tokens": 3108, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3311441100000252, "nonpadding_tokens": 8243, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3298420429999851, "nonpadding_tokens": 24652, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32835212300000194, "nonpadding_tokens": 18241, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.331315514000039, "nonpadding_tokens": 15919, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3321259080000232, "nonpadding_tokens": 17397, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.328905452000015, "nonpadding_tokens": 10379, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003738659999612537, "prefetch": {"data_wait_seconds": 0.006198330999737323, "consumed_batches": 656, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1206, "tokens": 97627772, "loss_tokens": 51673, "loss": 2.6462006270199137, "lr": 0.0002998381576102794, "optimizer_lrs": [0.019989210507351963, 0.0002998381576102794], "gradient_norm": 0.4394840598106384, "seconds": 0.27071962200000144, "nonpadding_tokens_per_second": 190872.75469082815, "ranks": [{"compute_seconds": 0.26682907100001785, "nonpadding_tokens": 9627, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26466162200000554, "nonpadding_tokens": 5582, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.265184454000007, "nonpadding_tokens": 10757, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2661360609999974, "nonpadding_tokens": 5241, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26836630600001854, "nonpadding_tokens": 1576, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2657807280000384, "nonpadding_tokens": 5515, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2680655139999999, "nonpadding_tokens": 7018, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2651151059999961, "nonpadding_tokens": 6357, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038284699996893323, "prefetch": {"data_wait_seconds": 0.006206767999756835, "consumed_batches": 657, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1207, "tokens": 97680771, "loss_tokens": 52999, "loss": 2.6702815147455614, "lr": 0.00029983793658897506, "optimizer_lrs": [0.019989195772598337, 0.00029983793658897506], "gradient_norm": 0.436047226190567, "seconds": 0.3606391760000065, "nonpadding_tokens_per_second": 146958.52122288302, "ranks": [{"compute_seconds": 0.3572301969999785, "nonpadding_tokens": 10202, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35829988199998297, "nonpadding_tokens": 8030, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3573387679999769, "nonpadding_tokens": 4612, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.35791001899997354, "nonpadding_tokens": 3489, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35809286400001383, "nonpadding_tokens": 12198, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3567929969999568, "nonpadding_tokens": 3358, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3541191869999807, "nonpadding_tokens": 7438, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3549571500000184, "nonpadding_tokens": 3672, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039487999998755186, "prefetch": {"data_wait_seconds": 0.006215278999789575, "consumed_batches": 658, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1208, "tokens": 97772060, "loss_tokens": 91289, "loss": 2.894835700358203, "lr": 0.0002998375555337869, "optimizer_lrs": [0.01998917036891913, 0.0002998375555337869], "gradient_norm": 0.40289750695228577, "seconds": 0.3209434729999998, "nonpadding_tokens_per_second": 284439.49692038156, "ranks": [{"compute_seconds": 0.31780670699998836, "nonpadding_tokens": 6398, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3181419340000389, "nonpadding_tokens": 2339, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3152071190000356, "nonpadding_tokens": 15636, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3176930409999841, "nonpadding_tokens": 14411, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31827719099999285, "nonpadding_tokens": 13094, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3171235020000154, "nonpadding_tokens": 22856, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31416893099998333, "nonpadding_tokens": 10805, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3186439030000088, "nonpadding_tokens": 5750, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039741099999446305, "prefetch": {"data_wait_seconds": 0.006223750999780009, "consumed_batches": 659, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1209, "tokens": 97836526, "loss_tokens": 64466, "loss": 2.8526628862501164, "lr": 0.0002998372861727811, "optimizer_lrs": [0.019989152411518742, 0.0002998372861727811], "gradient_norm": 0.6308412551879883, "seconds": 0.2935044079999898, "nonpadding_tokens_per_second": 219642.3571260376, "ranks": [{"compute_seconds": 0.23992018100000223, "nonpadding_tokens": 14099, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2611895090000189, "nonpadding_tokens": 2820, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2576682700000106, "nonpadding_tokens": 16430, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25995835100002296, "nonpadding_tokens": 3083, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25680111999997735, "nonpadding_tokens": 1626, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2580419290000009, "nonpadding_tokens": 4247, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26056954399996357, "nonpadding_tokens": 10877, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2610687279999979, "nonpadding_tokens": 11284, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0289545320000002, "prefetch": {"data_wait_seconds": 0.006231809999817983, "consumed_batches": 660, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1210, "tokens": 97904979, "loss_tokens": 68453, "loss": 2.7973527456795173, "lr": 0.00029983699990860665, "optimizer_lrs": [0.019989133327240446, 0.00029983699990860665], "gradient_norm": 0.39959055185317993, "seconds": 0.2889846539999894, "nonpadding_tokens_per_second": 236874.16979588996, "ranks": [{"compute_seconds": 0.2864195230000064, "nonpadding_tokens": 10672, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2866381370000113, "nonpadding_tokens": 3898, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2835737419999873, "nonpadding_tokens": 10142, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28646779799998967, "nonpadding_tokens": 1531, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28311929499994903, "nonpadding_tokens": 28620, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2844762579999838, "nonpadding_tokens": 2195, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.286605517000055, "nonpadding_tokens": 6541, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2834041730000081, "nonpadding_tokens": 4854, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004068819999929474, "prefetch": {"data_wait_seconds": 0.006241158999856111, "consumed_batches": 661, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1211, "tokens": 97960092, "loss_tokens": 55113, "loss": 3.3014579137408595, "lr": 0.0002998367692483754, "optimizer_lrs": [0.019989117949891694, 0.0002998367692483754], "gradient_norm": 0.486841082572937, "seconds": 0.25568443800000296, "nonpadding_tokens_per_second": 215550.85804635228, "ranks": [{"compute_seconds": 0.23793394299997317, "nonpadding_tokens": 5681, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24942472899999757, "nonpadding_tokens": 10954, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2486643029999982, "nonpadding_tokens": 5917, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24752215600000227, "nonpadding_tokens": 5119, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24547652000001108, "nonpadding_tokens": 3297, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24881637099997533, "nonpadding_tokens": 2658, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2457267999999999, "nonpadding_tokens": 19265, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24987666100003025, "nonpadding_tokens": 2222, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039944600001717845, "prefetch": {"data_wait_seconds": 0.006250333999858526, "consumed_batches": 662, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1212, "tokens": 98041066, "loss_tokens": 80974, "loss": 2.718252540939067, "lr": 0.0002998364300584228, "optimizer_lrs": [0.019989095337228188, 0.0002998364300584228], "gradient_norm": 0.4340968728065491, "seconds": 0.3040011450000293, "nonpadding_tokens_per_second": 266360.8388711569, "ranks": [{"compute_seconds": 0.30175108900004943, "nonpadding_tokens": 13525, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2999539609999715, "nonpadding_tokens": 6679, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2976831849999826, "nonpadding_tokens": 7769, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3011098549999929, "nonpadding_tokens": 8957, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2955613690000405, "nonpadding_tokens": 3227, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2985472870000194, "nonpadding_tokens": 12259, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2956736200000023, "nonpadding_tokens": 13096, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30146009500003856, "nonpadding_tokens": 15462, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003972160000103031, "prefetch": {"data_wait_seconds": 0.006260850999822765, "consumed_batches": 663, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1213, "tokens": 98108578, "loss_tokens": 67512, "loss": 2.6326537319291385, "lr": 0.0002998361469901452, "optimizer_lrs": [0.01998907646600968, 0.0002998361469901452], "gradient_norm": 0.5198378562927246, "seconds": 0.32353309999996327, "nonpadding_tokens_per_second": 208671.07569521532, "ranks": [{"compute_seconds": 0.32012501099995916, "nonpadding_tokens": 8288, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3208069659999637, "nonpadding_tokens": 4380, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31789899900002183, "nonpadding_tokens": 5014, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32030150500003174, "nonpadding_tokens": 10018, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3211597709999978, "nonpadding_tokens": 3436, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.318055615999981, "nonpadding_tokens": 13404, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31684912999998005, "nonpadding_tokens": 15201, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3212067299999717, "nonpadding_tokens": 7771, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003981990000170299, "prefetch": {"data_wait_seconds": 0.006269796999859523, "consumed_batches": 664, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1214, "tokens": 98167423, "loss_tokens": 58845, "loss": 2.9549430707791657, "lr": 0.0002998359000618818, "optimizer_lrs": [0.019989060004125453, 0.0002998359000618818], "gradient_norm": 0.4571569561958313, "seconds": 0.2675251000000003, "nonpadding_tokens_per_second": 219960.66911104767, "ranks": [{"compute_seconds": 0.2652899180000077, "nonpadding_tokens": 14170, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.261927691999972, "nonpadding_tokens": 5939, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.262536984999997, "nonpadding_tokens": 7342, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26474714300002233, "nonpadding_tokens": 7879, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2616631639999696, "nonpadding_tokens": 3982, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2629493180000395, "nonpadding_tokens": 11616, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2652564499999812, "nonpadding_tokens": 4742, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26224422599995023, "nonpadding_tokens": 3175, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003672160000292024, "prefetch": {"data_wait_seconds": 0.00627800399986711, "consumed_batches": 665, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1215, "tokens": 98214033, "loss_tokens": 46610, "loss": 2.6393058758850034, "lr": 0.00029983570434284786, "optimizer_lrs": [0.01998904695618986, 0.00029983570434284786], "gradient_norm": 0.4615837335586548, "seconds": 0.3301772430000369, "nonpadding_tokens_per_second": 141166.60365958288, "ranks": [{"compute_seconds": 0.32521042800004807, "nonpadding_tokens": 6844, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3236657079999645, "nonpadding_tokens": 2427, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32449870800002145, "nonpadding_tokens": 6462, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3250380380000024, "nonpadding_tokens": 6205, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32772845900001357, "nonpadding_tokens": 11841, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.324403808999989, "nonpadding_tokens": 3640, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3233750040000132, "nonpadding_tokens": 4265, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32789302099996576, "nonpadding_tokens": 4926, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034799500002691275, "prefetch": {"data_wait_seconds": 0.006286812999874769, "consumed_batches": 666, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1216, "tokens": 98271776, "loss_tokens": 57743, "loss": 2.4149071640718356, "lr": 0.00029983546171381, "optimizer_lrs": [0.01998903078092067, 0.00029983546171381], "gradient_norm": 0.5466613173484802, "seconds": 0.24655238999997664, "nonpadding_tokens_per_second": 234201.74511391055, "ranks": [{"compute_seconds": 0.24429416700002093, "nonpadding_tokens": 9401, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24008073600003854, "nonpadding_tokens": 5789, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24114102800001547, "nonpadding_tokens": 5831, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24169189200000574, "nonpadding_tokens": 5664, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24371307400002706, "nonpadding_tokens": 10653, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2423912140000084, "nonpadding_tokens": 2959, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2399425179999639, "nonpadding_tokens": 7928, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2440844200000356, "nonpadding_tokens": 9518, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003584680000017215, "prefetch": {"data_wait_seconds": 0.006294235999874331, "consumed_batches": 667, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1217, "tokens": 98339193, "loss_tokens": 67417, "loss": 2.8172940893988163, "lr": 0.00029983517820941866, "optimizer_lrs": [0.019989011880627912, 0.00029983517820941866], "gradient_norm": 0.6905497908592224, "seconds": 0.415614005000009, "nonpadding_tokens_per_second": 162210.60693081922, "ranks": [{"compute_seconds": 0.4125350390000335, "nonpadding_tokens": 12947, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4088699159999578, "nonpadding_tokens": 4235, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41003985500003637, "nonpadding_tokens": 5312, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4106174389999637, "nonpadding_tokens": 3813, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.40882844600002954, "nonpadding_tokens": 6364, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41127279599999156, "nonpadding_tokens": 7572, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.40887394300000324, "nonpadding_tokens": 1945, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.41321954000000005, "nonpadding_tokens": 25229, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004142580000348062, "prefetch": {"data_wait_seconds": 0.00630323199989391, "consumed_batches": 668, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1218, "tokens": 98419159, "loss_tokens": 79966, "loss": 3.3260737375884752, "lr": 0.00029983484161731774, "optimizer_lrs": [0.019988989441154518, 0.00029983484161731774], "gradient_norm": 1.9099704027175903, "seconds": 0.3786458149999703, "nonpadding_tokens_per_second": 211189.44626393475, "ranks": [{"compute_seconds": 0.3729362010000159, "nonpadding_tokens": 18191, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37559308100003364, "nonpadding_tokens": 24171, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37455886599997257, "nonpadding_tokens": 1296, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.37302998000001253, "nonpadding_tokens": 7847, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.37133051000000705, "nonpadding_tokens": 10614, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3726929219999988, "nonpadding_tokens": 6225, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.37147619600000326, "nonpadding_tokens": 7510, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37624322700003177, "nonpadding_tokens": 4112, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040025900000273396, "prefetch": {"data_wait_seconds": 0.006312545999904273, "consumed_batches": 669, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1219, "tokens": 98508193, "loss_tokens": 89034, "loss": 2.9526483562459287, "lr": 0.0002998344664525832, "optimizer_lrs": [0.019988964430172215, 0.0002998344664525832], "gradient_norm": 0.3911500871181488, "seconds": 0.39302443100001483, "nonpadding_tokens_per_second": 226535.5356496824, "ranks": [{"compute_seconds": 0.39063046400002577, "nonpadding_tokens": 13260, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.38998198100000536, "nonpadding_tokens": 7143, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38950426599996035, "nonpadding_tokens": 8743, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.38809985499995037, "nonpadding_tokens": 3025, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.38656284799998275, "nonpadding_tokens": 17819, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38954877099996565, "nonpadding_tokens": 19928, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3860925669999915, "nonpadding_tokens": 4071, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.390655253000034, "nonpadding_tokens": 15045, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003963400000088768, "prefetch": {"data_wait_seconds": 0.006330670999901145, "consumed_batches": 670, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1220, "tokens": 98569596, "loss_tokens": 61403, "loss": 2.803618104978584, "lr": 0.00029983420746943504, "optimizer_lrs": [0.019988947164629005, 0.00029983420746943504], "gradient_norm": 0.4641008675098419, "seconds": 0.34328853900001377, "nonpadding_tokens_per_second": 178867.02591022864, "ranks": [{"compute_seconds": 0.3385849050000047, "nonpadding_tokens": 14462, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33770961999999827, "nonpadding_tokens": 10728, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3378992600000288, "nonpadding_tokens": 5936, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3386360229999923, "nonpadding_tokens": 4102, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.340971852999985, "nonpadding_tokens": 3178, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33969794000000775, "nonpadding_tokens": 3684, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3407698120000191, "nonpadding_tokens": 10343, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3374762460000511, "nonpadding_tokens": 8970, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035531700001456556, "prefetch": {"data_wait_seconds": 0.006340229999935332, "consumed_batches": 671, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1221, "tokens": 98688934, "loss_tokens": 119338, "loss": 2.6286115906081884, "lr": 0.00029983370355164164, "optimizer_lrs": [0.019988913570109443, 0.00029983370355164164], "gradient_norm": 0.5149298310279846, "seconds": 0.34625030600000173, "nonpadding_tokens_per_second": 344658.17916128977, "ranks": [{"compute_seconds": 0.3439178730000094, "nonpadding_tokens": 16236, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.340124306000007, "nonpadding_tokens": 17027, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3409943450000128, "nonpadding_tokens": 20619, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34179843399999754, "nonpadding_tokens": 16662, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33981126399999084, "nonpadding_tokens": 17453, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3427007930000059, "nonpadding_tokens": 17468, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3438587150000103, "nonpadding_tokens": 4780, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34026655799999617, "nonpadding_tokens": 9093, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038657800001828946, "prefetch": {"data_wait_seconds": 0.006348977999962813, "consumed_batches": 672, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1222, "tokens": 98750958, "loss_tokens": 62024, "loss": 2.660617764897459, "lr": 0.0002998334413467015, "optimizer_lrs": [0.0199888960897801, 0.0002998334413467015], "gradient_norm": 0.4559832811355591, "seconds": 0.2577087639999718, "nonpadding_tokens_per_second": 240674.77969048344, "ranks": [{"compute_seconds": 0.2540809169999534, "nonpadding_tokens": 3538, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25534314100002575, "nonpadding_tokens": 3593, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2522680170000058, "nonpadding_tokens": 19715, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25488439299999754, "nonpadding_tokens": 10740, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25121407500000714, "nonpadding_tokens": 3257, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2524483299999929, "nonpadding_tokens": 7641, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2548508390000279, "nonpadding_tokens": 1534, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25544558600000755, "nonpadding_tokens": 12006, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033660999997664476, "prefetch": {"data_wait_seconds": 0.006357165999943959, "consumed_batches": 673, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1223, "tokens": 98872245, "loss_tokens": 121287, "loss": 2.7611383536570284, "lr": 0.00029983292801238003, "optimizer_lrs": [0.019988861867492002, 0.00029983292801238003], "gradient_norm": 0.49681326746940613, "seconds": 0.28889160200003516, "nonpadding_tokens_per_second": 419835.6724816986, "ranks": [{"compute_seconds": 0.2833577380000065, "nonpadding_tokens": 6915, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2860221010000146, "nonpadding_tokens": 2630, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2826372659999947, "nonpadding_tokens": 11623, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2859140909999951, "nonpadding_tokens": 20332, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2865534709999906, "nonpadding_tokens": 18954, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28393134900005634, "nonpadding_tokens": 17752, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2860391510000113, "nonpadding_tokens": 25116, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28277172800000017, "nonpadding_tokens": 17965, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037205699999276476, "prefetch": {"data_wait_seconds": 0.006365100999985884, "consumed_batches": 674, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1224, "tokens": 98957476, "loss_tokens": 85231, "loss": 2.4064602741373444, "lr": 0.00029983256680900195, "optimizer_lrs": [0.0199888377872668, 0.00029983256680900195], "gradient_norm": 0.42012807726860046, "seconds": 0.26035385499994845, "nonpadding_tokens_per_second": 327365.99963160476, "ranks": [{"compute_seconds": 0.2551553620000391, "nonpadding_tokens": 6558, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25706717999997863, "nonpadding_tokens": 16707, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2541618670000503, "nonpadding_tokens": 10357, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2548352160000036, "nonpadding_tokens": 2054, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25773546099998157, "nonpadding_tokens": 5523, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25459777100002157, "nonpadding_tokens": 24802, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.25308042699998623, "nonpadding_tokens": 13648, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25803229699999974, "nonpadding_tokens": 5582, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003636729999811905, "prefetch": {"data_wait_seconds": 0.006373237999994785, "consumed_batches": 675, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1225, "tokens": 99020910, "loss_tokens": 63434, "loss": 2.5238675729892486, "lr": 0.00029983229772686826, "optimizer_lrs": [0.019988819848457885, 0.00029983229772686826], "gradient_norm": 0.3979116380214691, "seconds": 0.28595638199999485, "nonpadding_tokens_per_second": 221831.03435684516, "ranks": [{"compute_seconds": 0.2822018789999561, "nonpadding_tokens": 7302, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2801844210000013, "nonpadding_tokens": 2395, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2825631460000295, "nonpadding_tokens": 21114, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2817392320000067, "nonpadding_tokens": 9602, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27989076199997953, "nonpadding_tokens": 2465, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2808096379999938, "nonpadding_tokens": 10345, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2837101640000128, "nonpadding_tokens": 3756, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28045066699996823, "nonpadding_tokens": 6455, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033459200000152123, "prefetch": {"data_wait_seconds": 0.006382249000026263, "consumed_batches": 676, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1226, "tokens": 99068327, "loss_tokens": 47417, "loss": 3.032351925469768, "lr": 0.0002998320964465934, "optimizer_lrs": [0.019988806429772896, 0.0002998320964465934], "gradient_norm": 0.395283043384552, "seconds": 0.273131750999994, "nonpadding_tokens_per_second": 173604.8622190433, "ranks": [{"compute_seconds": 0.27053848399998515, "nonpadding_tokens": 3923, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2672621489999756, "nonpadding_tokens": 5237, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2700687699999662, "nonpadding_tokens": 7104, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2704862250000133, "nonpadding_tokens": 3258, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2669530430000009, "nonpadding_tokens": 3282, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26972156799996583, "nonpadding_tokens": 9178, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2708220750000123, "nonpadding_tokens": 3111, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2672588870000254, "nonpadding_tokens": 12324, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003298739999877398, "prefetch": {"data_wait_seconds": 0.006390045000003397, "consumed_batches": 677, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1227, "tokens": 99137040, "loss_tokens": 68713, "loss": 2.8823209854758196, "lr": 0.00029983180455290913, "optimizer_lrs": [0.019988786970193944, 0.00029983180455290913], "gradient_norm": 0.39104872941970825, "seconds": 0.37516080400001783, "nonpadding_tokens_per_second": 183156.12736557823, "ranks": [{"compute_seconds": 0.3708506020000186, "nonpadding_tokens": 18209, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3726181970000084, "nonpadding_tokens": 11148, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36958670299998175, "nonpadding_tokens": 9717, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3702102790000481, "nonpadding_tokens": 7477, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3720997180000154, "nonpadding_tokens": 6362, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37127296599999227, "nonpadding_tokens": 9353, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3727503740000202, "nonpadding_tokens": 4889, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3727521619999834, "nonpadding_tokens": 1558, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004550709999762148, "prefetch": {"data_wait_seconds": 0.006399125999976718, "consumed_batches": 678, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1228, "tokens": 99189145, "loss_tokens": 52105, "loss": 2.9321412292486326, "lr": 0.0002998315830413227, "optimizer_lrs": [0.01998877220275485, 0.0002998315830413227], "gradient_norm": 0.37810948491096497, "seconds": 0.28411547600001086, "nonpadding_tokens_per_second": 183393.74093088123, "ranks": [{"compute_seconds": 0.2815653140000336, "nonpadding_tokens": 10458, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2780479940000191, "nonpadding_tokens": 2650, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27876566700001604, "nonpadding_tokens": 4725, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28140919800000574, "nonpadding_tokens": 9615, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2777894309999738, "nonpadding_tokens": 3203, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28063284500001373, "nonpadding_tokens": 8771, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28137123099998007, "nonpadding_tokens": 10471, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.278590027000007, "nonpadding_tokens": 2212, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004298879999851124, "prefetch": {"data_wait_seconds": 0.0064078589999923, "consumed_batches": 679, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1229, "tokens": 99255760, "loss_tokens": 66615, "loss": 2.8424805786984915, "lr": 0.0002998312996318645, "optimizer_lrs": [0.019988753308790966, 0.0002998312996318645], "gradient_norm": 0.4160054624080658, "seconds": 0.547588152000003, "nonpadding_tokens_per_second": 121651.64596183527, "ranks": [{"compute_seconds": 0.5440489309999634, "nonpadding_tokens": 5345, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5439945829999715, "nonpadding_tokens": 4448, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5408075440000175, "nonpadding_tokens": 1736, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.5414049120000186, "nonpadding_tokens": 10233, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.544776818999992, "nonpadding_tokens": 11497, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5421744469999794, "nonpadding_tokens": 18751, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.5450385940000047, "nonpadding_tokens": 10322, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.5451289810000048, "nonpadding_tokens": 4283, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044160999999576234, "prefetch": {"data_wait_seconds": 0.006417515999956436, "consumed_batches": 680, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1230, "tokens": 99311828, "loss_tokens": 56068, "loss": 2.8816863562994937, "lr": 0.0002998310609094305, "optimizer_lrs": [0.019988737393962036, 0.0002998310609094305], "gradient_norm": 0.43163928389549255, "seconds": 0.25886667599996827, "nonpadding_tokens_per_second": 216590.25744977262, "ranks": [{"compute_seconds": 0.2537310110000135, "nonpadding_tokens": 11882, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25538702099999, "nonpadding_tokens": 9204, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25264889400000357, "nonpadding_tokens": 6293, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2531737819999762, "nonpadding_tokens": 3022, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2517850220000355, "nonpadding_tokens": 9261, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25466827600001807, "nonpadding_tokens": 2090, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2519376189999889, "nonpadding_tokens": 4688, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2564615209999488, "nonpadding_tokens": 9628, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003668720000291614, "prefetch": {"data_wait_seconds": 0.006427153999950974, "consumed_batches": 681, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1231, "tokens": 99371395, "loss_tokens": 59567, "loss": 2.66934213784478, "lr": 0.00029983080710440003, "optimizer_lrs": [0.01998872047362667, 0.00029983080710440003], "gradient_norm": 0.4865912199020386, "seconds": 0.25514234700000316, "nonpadding_tokens_per_second": 233465.75235509322, "ranks": [{"compute_seconds": 0.2529116869999939, "nonpadding_tokens": 12718, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25266860100003896, "nonpadding_tokens": 8288, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24974723100001484, "nonpadding_tokens": 2749, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24997584700003017, "nonpadding_tokens": 11534, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24857728200004203, "nonpadding_tokens": 4776, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25003135999997994, "nonpadding_tokens": 1868, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2525179849999972, "nonpadding_tokens": 4410, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25273943700000245, "nonpadding_tokens": 13224, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036270799995463676, "prefetch": {"data_wait_seconds": 0.006440236999949889, "consumed_batches": 682, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1232, "tokens": 99418767, "loss_tokens": 47372, "loss": 2.6376160693553152, "lr": 0.00029983060512431405, "optimizer_lrs": [0.019988707008287605, 0.00029983060512431405], "gradient_norm": 1.2608381509780884, "seconds": 0.5150040739999895, "nonpadding_tokens_per_second": 91983.73836553567, "ranks": [{"compute_seconds": 0.5127036119999957, "nonpadding_tokens": 11554, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.508349389999978, "nonpadding_tokens": 2913, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5091625130000352, "nonpadding_tokens": 3540, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.5098463930000321, "nonpadding_tokens": 1829, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.5122858299999962, "nonpadding_tokens": 13926, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5108972720000224, "nonpadding_tokens": 2842, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.511655243000007, "nonpadding_tokens": 5456, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.509077384999955, "nonpadding_tokens": 5312, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003968990000089434, "prefetch": {"data_wait_seconds": 0.006448899999966784, "consumed_batches": 683, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1233, "tokens": 99501292, "loss_tokens": 82525, "loss": 2.8780672523477735, "lr": 0.0002998302529747187, "optimizer_lrs": [0.019988683531647917, 0.0002998302529747187], "gradient_norm": 0.4581458270549774, "seconds": 0.3595604670000512, "nonpadding_tokens_per_second": 229516.33334036206, "ranks": [{"compute_seconds": 0.3572680339999579, "nonpadding_tokens": 10569, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35693654699997523, "nonpadding_tokens": 10239, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35325278599998455, "nonpadding_tokens": 11428, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3567902570000001, "nonpadding_tokens": 3459, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3562795940000001, "nonpadding_tokens": 13402, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3536996589999717, "nonpadding_tokens": 9705, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3524404219999724, "nonpadding_tokens": 20513, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35312608899999987, "nonpadding_tokens": 3210, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003916580000122849, "prefetch": {"data_wait_seconds": 0.006458508999983792, "consumed_batches": 684, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1234, "tokens": 99553739, "loss_tokens": 52447, "loss": 2.723761011115984, "lr": 0.0002998300289836531, "optimizer_lrs": [0.019988668598910207, 0.0002998300289836531], "gradient_norm": 0.435030460357666, "seconds": 0.25146473900002775, "nonpadding_tokens_per_second": 208566.02086065916, "ranks": [{"compute_seconds": 0.24671812999997655, "nonpadding_tokens": 4033, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24868457999997418, "nonpadding_tokens": 12391, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24549738300004265, "nonpadding_tokens": 8739, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24792335099999718, "nonpadding_tokens": 3639, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24452042600000823, "nonpadding_tokens": 3559, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24556544799997937, "nonpadding_tokens": 2189, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24801723899997796, "nonpadding_tokens": 6741, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24915694100002383, "nonpadding_tokens": 11156, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037834699998029464, "prefetch": {"data_wait_seconds": 0.006467393999983528, "consumed_batches": 685, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1235, "tokens": 99589194, "loss_tokens": 35455, "loss": 2.9322512692144973, "lr": 0.00029982987747854343, "optimizer_lrs": [0.019988658498569564, 0.00029982987747854343], "gradient_norm": 0.49097368121147156, "seconds": 0.22236311199998227, "nonpadding_tokens_per_second": 159446.41033807275, "ranks": [{"compute_seconds": 0.21759521100000256, "nonpadding_tokens": 4812, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21947230200004242, "nonpadding_tokens": 3679, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21634296700000277, "nonpadding_tokens": 2654, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2192681909999692, "nonpadding_tokens": 4586, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21958659500000977, "nonpadding_tokens": 5161, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21818809400002692, "nonpadding_tokens": 8462, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.21524579699996593, "nonpadding_tokens": 3770, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22008581200003619, "nonpadding_tokens": 2331, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003339570000093772, "prefetch": {"data_wait_seconds": 0.006475363999996375, "consumed_batches": 686, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1236, "tokens": 99705590, "loss_tokens": 116396, "loss": 2.434373013247878, "lr": 0.0002998293796248937, "optimizer_lrs": [0.019988625308326246, 0.0002998293796248937], "gradient_norm": 0.6451997756958008, "seconds": 0.29469208599999774, "nonpadding_tokens_per_second": 394974.97737350466, "ranks": [{"compute_seconds": 0.2923493930000518, "nonpadding_tokens": 11517, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2880557330000215, "nonpadding_tokens": 17356, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2880202869999948, "nonpadding_tokens": 11430, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28879585399999996, "nonpadding_tokens": 17975, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.287037764000047, "nonpadding_tokens": 10923, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28982486500001414, "nonpadding_tokens": 19546, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28709309900000335, "nonpadding_tokens": 16663, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29173786400002655, "nonpadding_tokens": 10986, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036898799999107723, "prefetch": {"data_wait_seconds": 0.006484144000026504, "consumed_batches": 687, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1237, "tokens": 99856147, "loss_tokens": 150557, "loss": 2.1967691223257635, "lr": 0.00029982873457816243, "optimizer_lrs": [0.01998858230521083, 0.00029982873457816243], "gradient_norm": 0.4879896938800812, "seconds": 0.35365138200000956, "nonpadding_tokens_per_second": 425721.5089859197, "ranks": [{"compute_seconds": 0.3481675890000133, "nonpadding_tokens": 13423, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3491438540000331, "nonpadding_tokens": 17385, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34618964300000243, "nonpadding_tokens": 18019, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3505681820000177, "nonpadding_tokens": 19202, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35101229800000056, "nonpadding_tokens": 27863, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3478991009999959, "nonpadding_tokens": 26212, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.35078738900000417, "nonpadding_tokens": 18498, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3513128970000139, "nonpadding_tokens": 9955, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036318799999435214, "prefetch": {"data_wait_seconds": 0.006493804000015757, "consumed_batches": 688, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1238, "tokens": 99939336, "loss_tokens": 83189, "loss": 2.666086787315631, "lr": 0.0002998283776413541, "optimizer_lrs": [0.019988558509423607, 0.0002998283776413541], "gradient_norm": 0.4330425262451172, "seconds": 0.27098935800000845, "nonpadding_tokens_per_second": 306982.5347163537, "ranks": [{"compute_seconds": 0.26809139599998844, "nonpadding_tokens": 11205, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2677217709999695, "nonpadding_tokens": 19653, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26520485200001076, "nonpadding_tokens": 21306, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26788587600003666, "nonpadding_tokens": 2630, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2686486129999821, "nonpadding_tokens": 4134, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26560017700001026, "nonpadding_tokens": 5765, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26783952400001, "nonpadding_tokens": 11728, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2651048330000094, "nonpadding_tokens": 6768, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036716700003580627, "prefetch": {"data_wait_seconds": 0.006502818000015509, "consumed_batches": 689, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1239, "tokens": 99996236, "loss_tokens": 56900, "loss": 2.821517464850615, "lr": 0.0002998281332882023, "optimizer_lrs": [0.019988542219213486, 0.0002998281332882023], "gradient_norm": 0.4742518663406372, "seconds": 0.252280202999998, "nonpadding_tokens_per_second": 225542.86592198617, "ranks": [{"compute_seconds": 0.24627532199997404, "nonpadding_tokens": 3928, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24519106299999294, "nonpadding_tokens": 11362, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24559670799999367, "nonpadding_tokens": 8435, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24611231800002997, "nonpadding_tokens": 2997, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24502070500000173, "nonpadding_tokens": 13370, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24590870899999118, "nonpadding_tokens": 4549, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24442290000001776, "nonpadding_tokens": 8878, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2497265450000441, "nonpadding_tokens": 3381, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038151299997934984, "prefetch": {"data_wait_seconds": 0.006511250000031055, "consumed_batches": 690, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1240, "tokens": 100064817, "loss_tokens": 68581, "loss": 2.578359895597906, "lr": 0.0002998278385409232, "optimizer_lrs": [0.01998852256939488, 0.0002998278385409232], "gradient_norm": 0.459040105342865, "seconds": 0.23901857099997414, "nonpadding_tokens_per_second": 286927.4956882217, "ranks": [{"compute_seconds": 0.23582748999996284, "nonpadding_tokens": 4375, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23644487699999672, "nonpadding_tokens": 3626, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2338815020000311, "nonpadding_tokens": 15469, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23675146199997243, "nonpadding_tokens": 12848, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23278549599996268, "nonpadding_tokens": 8189, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23388803699998562, "nonpadding_tokens": 5415, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23648045400000228, "nonpadding_tokens": 6927, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23339385300005233, "nonpadding_tokens": 11732, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003764600000408791, "prefetch": {"data_wait_seconds": 0.006522889999999393, "consumed_batches": 691, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1241, "tokens": 100138375, "loss_tokens": 73558, "loss": 2.6281359182549826, "lr": 0.0002998275221230032, "optimizer_lrs": [0.01998850147486688, 0.0002998275221230032], "gradient_norm": 0.476285845041275, "seconds": 0.2781036630000244, "nonpadding_tokens_per_second": 264498.49385836226, "ranks": [{"compute_seconds": 0.2750877370000353, "nonpadding_tokens": 6468, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27535026200001766, "nonpadding_tokens": 6806, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27223389300002054, "nonpadding_tokens": 12094, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2747345300000461, "nonpadding_tokens": 2735, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27563324599998396, "nonpadding_tokens": 8082, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2744943519999765, "nonpadding_tokens": 6446, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2717989459999899, "nonpadding_tokens": 6152, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27580511100001104, "nonpadding_tokens": 24775, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003568779999909566, "prefetch": {"data_wait_seconds": 0.006531751000025565, "consumed_batches": 692, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1242, "tokens": 100193154, "loss_tokens": 54779, "loss": 2.557848069059311, "lr": 0.00029982728629643476, "optimizer_lrs": [0.019988485753095652, 0.00029982728629643476], "gradient_norm": 0.40932297706604004, "seconds": 0.32640537899999345, "nonpadding_tokens_per_second": 167825.05290760266, "ranks": [{"compute_seconds": 0.3241003289999753, "nonpadding_tokens": 12350, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3231192579999629, "nonpadding_tokens": 4888, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31981957900001134, "nonpadding_tokens": 5902, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.323814896999977, "nonpadding_tokens": 7180, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32401160799997797, "nonpadding_tokens": 9762, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32142985600000884, "nonpadding_tokens": 3330, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3237421819999895, "nonpadding_tokens": 7256, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3240870010000094, "nonpadding_tokens": 4111, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035007299999278985, "prefetch": {"data_wait_seconds": 0.006541515000051845, "consumed_batches": 693, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1243, "tokens": 100243333, "loss_tokens": 50179, "loss": 2.601271821877678, "lr": 0.0002998270701318211, "optimizer_lrs": [0.01998847134212141, 0.0002998270701318211], "gradient_norm": 0.43979549407958984, "seconds": 0.25195563900001616, "nonpadding_tokens_per_second": 199158.0748069575, "ranks": [{"compute_seconds": 0.21971540199996298, "nonpadding_tokens": 4933, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2195202199999926, "nonpadding_tokens": 12166, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21964861499998278, "nonpadding_tokens": 2036, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21729530699997213, "nonpadding_tokens": 3791, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21609471900001154, "nonpadding_tokens": 6206, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2179868460000307, "nonpadding_tokens": 11362, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.21575345399998014, "nonpadding_tokens": 3755, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2213323710000168, "nonpadding_tokens": 5930, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.02320496900000535, "prefetch": {"data_wait_seconds": 0.0065503040000436386, "consumed_batches": 694, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1244, "tokens": 100295986, "loss_tokens": 52653, "loss": 2.857476960002279, "lr": 0.000299826843164279, "optimizer_lrs": [0.019988456210951935, 0.000299826843164279], "gradient_norm": 0.4113742411136627, "seconds": 0.2758750309999982, "nonpadding_tokens_per_second": 190858.1570762029, "ranks": [{"compute_seconds": 0.27327229300004774, "nonpadding_tokens": 2891, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26977795699997387, "nonpadding_tokens": 4970, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2717419790000122, "nonpadding_tokens": 3995, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2703162100000327, "nonpadding_tokens": 11971, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2686781030000134, "nonpadding_tokens": 14885, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2715772130000005, "nonpadding_tokens": 3192, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27295437899999797, "nonpadding_tokens": 5008, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.269629067999972, "nonpadding_tokens": 5741, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003811810000229343, "prefetch": {"data_wait_seconds": 0.006559410000036792, "consumed_batches": 695, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1245, "tokens": 100345234, "loss_tokens": 49248, "loss": 3.1413801068063676, "lr": 0.00029982663073979427, "optimizer_lrs": [0.01998844204931962, 0.00029982663073979427], "gradient_norm": 0.4646984040737152, "seconds": 0.2597459770000228, "nonpadding_tokens_per_second": 189600.62661527065, "ranks": [{"compute_seconds": 0.25661358899998277, "nonpadding_tokens": 4459, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2574723999999833, "nonpadding_tokens": 3261, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2540815350000116, "nonpadding_tokens": 5434, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25684969799999635, "nonpadding_tokens": 2148, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25709161300000005, "nonpadding_tokens": 4748, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2557030709999708, "nonpadding_tokens": 19422, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.25704122899998083, "nonpadding_tokens": 6347, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2539827319999972, "nonpadding_tokens": 3429, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003782860000001165, "prefetch": {"data_wait_seconds": 0.006567065000027128, "consumed_batches": 696, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1246, "tokens": 100388940, "loss_tokens": 43706, "loss": 2.5134649704846015, "lr": 0.0002998264421109732, "optimizer_lrs": [0.019988429474064883, 0.0002998264421109732], "gradient_norm": 0.4417029619216919, "seconds": 0.2796356139999716, "nonpadding_tokens_per_second": 156296.25774349485, "ranks": [{"compute_seconds": 0.2769598090000045, "nonpadding_tokens": 6147, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2763404319999836, "nonpadding_tokens": 2653, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2734181749999607, "nonpadding_tokens": 6056, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2745202860000404, "nonpadding_tokens": 3881, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27233651000000236, "nonpadding_tokens": 2100, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27494020299997146, "nonpadding_tokens": 4146, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2760554430000184, "nonpadding_tokens": 4438, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2766239669999777, "nonpadding_tokens": 14285, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003964340000379707, "prefetch": {"data_wait_seconds": 0.006576592000001824, "consumed_batches": 697, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1247, "tokens": 100481816, "loss_tokens": 92876, "loss": 2.718674294220251, "lr": 0.000299826040931175, "optimizer_lrs": [0.019988402728745, 0.000299826040931175], "gradient_norm": 0.6344153881072998, "seconds": 0.37883065199997645, "nonpadding_tokens_per_second": 245164.95565941103, "ranks": [{"compute_seconds": 0.3756673439999645, "nonpadding_tokens": 21417, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37645009900001014, "nonpadding_tokens": 10812, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37463851100000056, "nonpadding_tokens": 8890, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3744159659999582, "nonpadding_tokens": 2089, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.37597342399999434, "nonpadding_tokens": 13148, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3753335119999974, "nonpadding_tokens": 4256, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3757776219999869, "nonpadding_tokens": 16094, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37630414099999143, "nonpadding_tokens": 16170, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039555499995458376, "prefetch": {"data_wait_seconds": 0.006585320000013439, "consumed_batches": 698, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1248, "tokens": 100534226, "loss_tokens": 52410, "loss": 2.429225249236787, "lr": 0.00029982581434082013, "optimizer_lrs": [0.019988387622721343, 0.00029982581434082013], "gradient_norm": 0.47673290967941284, "seconds": 0.2494414079999956, "nonpadding_tokens_per_second": 210109.46185807663, "ranks": [{"compute_seconds": 0.24655112300001747, "nonpadding_tokens": 5507, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24345137300002762, "nonpadding_tokens": 8053, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24458852600002956, "nonpadding_tokens": 4198, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24415222200002518, "nonpadding_tokens": 2002, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.246867543999997, "nonpadding_tokens": 17240, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24479036200000337, "nonpadding_tokens": 2488, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24656590200004302, "nonpadding_tokens": 2200, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2470757840000033, "nonpadding_tokens": 10722, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003751170000327875, "prefetch": {"data_wait_seconds": 0.006593604000002529, "consumed_batches": 699, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1249, "tokens": 100581162, "loss_tokens": 46936, "loss": 2.940244428583603, "lr": 0.00029982561129176193, "optimizer_lrs": [0.019988374086117464, 0.00029982561129176193], "gradient_norm": 0.4693129360675812, "seconds": 0.31075033000001895, "nonpadding_tokens_per_second": 151040.86936930087, "ranks": [{"compute_seconds": 0.3056441709999831, "nonpadding_tokens": 6081, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30822370999999293, "nonpadding_tokens": 7417, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30493015799999057, "nonpadding_tokens": 8150, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30748980300000994, "nonpadding_tokens": 5043, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3043584380000084, "nonpadding_tokens": 3768, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30531890899999325, "nonpadding_tokens": 2497, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3079248739999798, "nonpadding_tokens": 10635, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3084532590000322, "nonpadding_tokens": 3345, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036069700001917226, "prefetch": {"data_wait_seconds": 0.006603094999960035, "consumed_batches": 700, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1250, "tokens": 100653811, "loss_tokens": 72649, "loss": 2.7310890807168717, "lr": 0.00029982529677304536, "optimizer_lrs": [0.019988353118203024, 0.00029982529677304536], "gradient_norm": 0.4618636965751648, "seconds": 0.5024883209999871, "nonpadding_tokens_per_second": 144578.48464104277, "ranks": [{"compute_seconds": 0.49823112499996114, "nonpadding_tokens": 13492, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4962541520000059, "nonpadding_tokens": 5025, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.49657957300001954, "nonpadding_tokens": 6611, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4997787549999657, "nonpadding_tokens": 19028, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.49576391800002284, "nonpadding_tokens": 6206, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.49687371100003475, "nonpadding_tokens": 7914, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.4955057280000119, "nonpadding_tokens": 6373, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.5001201669999773, "nonpadding_tokens": 8000, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036917200003472317, "prefetch": {"data_wait_seconds": 0.006612847999988958, "consumed_batches": 701, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1251, "tokens": 100734613, "loss_tokens": 80802, "loss": 2.584111856760971, "lr": 0.0002998249466249732, "optimizer_lrs": [0.019988329774998214, 0.0002998249466249732], "gradient_norm": 0.40709757804870605, "seconds": 0.21097248699999227, "nonpadding_tokens_per_second": 382997.8076714949, "ranks": [{"compute_seconds": 0.207929088999947, "nonpadding_tokens": 4652, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20475688300001593, "nonpadding_tokens": 20733, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20753224600002795, "nonpadding_tokens": 10245, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.20826319400003968, "nonpadding_tokens": 23280, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.20423439699999335, "nonpadding_tokens": 3519, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20762870100003283, "nonpadding_tokens": 5846, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.20440876600002866, "nonpadding_tokens": 1217, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.208626277999997, "nonpadding_tokens": 11310, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037476400001423826, "prefetch": {"data_wait_seconds": 0.006621566999982065, "consumed_batches": 702, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1252, "tokens": 100784137, "loss_tokens": 49524, "loss": 2.522217132097569, "lr": 0.00029982473184409923, "optimizer_lrs": [0.019988315456273283, 0.00029982473184409923], "gradient_norm": 0.7317183017730713, "seconds": 0.25057412699999304, "nonpadding_tokens_per_second": 197642.11330566215, "ranks": [{"compute_seconds": 0.21571720899999036, "nonpadding_tokens": 4801, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21281576100000166, "nonpadding_tokens": 2293, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.213434674000041, "nonpadding_tokens": 7044, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21621130699998048, "nonpadding_tokens": 6124, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21238617099999146, "nonpadding_tokens": 8060, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2136397860000443, "nonpadding_tokens": 7578, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.21629627200002233, "nonpadding_tokens": 3017, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21726303399998415, "nonpadding_tokens": 10607, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.005376168999987385, "prefetch": {"data_wait_seconds": 0.006629931999967198, "consumed_batches": 703, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1253, "tokens": 100864628, "loss_tokens": 80491, "loss": 2.622229503919693, "lr": 0.00029982438248154304, "optimizer_lrs": [0.019988292165436204, 0.00029982438248154304], "gradient_norm": 0.6462591886520386, "seconds": 0.31171710600000324, "nonpadding_tokens_per_second": 258218.1036930298, "ranks": [{"compute_seconds": 0.300361568000028, "nonpadding_tokens": 9423, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3028921459999765, "nonpadding_tokens": 11611, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3019009469999787, "nonpadding_tokens": 8640, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30225096599997414, "nonpadding_tokens": 14140, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2998729649999632, "nonpadding_tokens": 4545, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3019602780000241, "nonpadding_tokens": 11927, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2994653220000032, "nonpadding_tokens": 17885, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30311831099999154, "nonpadding_tokens": 2320, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000333248000004005, "prefetch": {"data_wait_seconds": 0.006637010999966719, "consumed_batches": 704, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1254, "tokens": 100936854, "loss_tokens": 72226, "loss": 2.243848735220004, "lr": 0.00029982406869642265, "optimizer_lrs": [0.01998827124642818, 0.00029982406869642265], "gradient_norm": 0.6022631525993347, "seconds": 0.32236649800000805, "nonpadding_tokens_per_second": 224049.33654116315, "ranks": [{"compute_seconds": 0.3194049489999884, "nonpadding_tokens": 10080, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3193590879999988, "nonpadding_tokens": 13169, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31670359000003145, "nonpadding_tokens": 12207, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31717147699998804, "nonpadding_tokens": 4892, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3154102059999673, "nonpadding_tokens": 2966, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3170975279999766, "nonpadding_tokens": 2634, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3193195139999716, "nonpadding_tokens": 1372, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3199713149999752, "nonpadding_tokens": 24906, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004041479999727926, "prefetch": {"data_wait_seconds": 0.0066465159999467005, "consumed_batches": 705, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1255, "tokens": 101047700, "loss_tokens": 110846, "loss": 2.1552116788156543, "lr": 0.0002998235865827035, "optimizer_lrs": [0.019988239105513568, 0.0002998235865827035], "gradient_norm": 0.5474215149879456, "seconds": 0.2764126750000173, "nonpadding_tokens_per_second": 401016.3426839708, "ranks": [{"compute_seconds": 0.27306063300000005, "nonpadding_tokens": 2387, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2742065040000057, "nonpadding_tokens": 11546, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.271431113999995, "nonpadding_tokens": 18267, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2729220010000404, "nonpadding_tokens": 25629, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2703257779999717, "nonpadding_tokens": 26302, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27096062300000767, "nonpadding_tokens": 20678, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2729986790000112, "nonpadding_tokens": 4259, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2735472489999893, "nonpadding_tokens": 1778, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036203999997042047, "prefetch": {"data_wait_seconds": 0.0066561159999452, "consumed_batches": 706, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1256, "tokens": 101125111, "loss_tokens": 77411, "loss": 2.8963261035253387, "lr": 0.0002998232495002528, "optimizer_lrs": [0.019988216633350188, 0.0002998232495002528], "gradient_norm": 0.39815667271614075, "seconds": 0.23067500000001928, "nonpadding_tokens_per_second": 335584.69708461483, "ranks": [{"compute_seconds": 0.22604904500002476, "nonpadding_tokens": 3755, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2282708089999801, "nonpadding_tokens": 13108, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22611317199999803, "nonpadding_tokens": 7865, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22792128399999, "nonpadding_tokens": 5907, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.22835680399998637, "nonpadding_tokens": 4898, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22578988600002958, "nonpadding_tokens": 18295, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.22776677100000597, "nonpadding_tokens": 17881, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22482306900002413, "nonpadding_tokens": 5702, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000392460000000483, "prefetch": {"data_wait_seconds": 0.006665238999929102, "consumed_batches": 707, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1257, "tokens": 101181897, "loss_tokens": 56786, "loss": 2.564649242330856, "lr": 0.00029982300202392856, "optimizer_lrs": [0.019988200134928573, 0.00029982300202392856], "gradient_norm": 1.619760274887085, "seconds": 0.4136579870000219, "nonpadding_tokens_per_second": 137277.65880173128, "ranks": [{"compute_seconds": 0.40997965700000805, "nonpadding_tokens": 3664, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.409130690999973, "nonpadding_tokens": 15413, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4111798019999924, "nonpadding_tokens": 3621, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.40679264300001705, "nonpadding_tokens": 3443, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.40532524999997577, "nonpadding_tokens": 10042, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4091078019999941, "nonpadding_tokens": 6471, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.4050711159999878, "nonpadding_tokens": 11300, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.40589037400002326, "nonpadding_tokens": 2832, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003654960000289975, "prefetch": {"data_wait_seconds": 0.006674675999889246, "consumed_batches": 708, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1258, "tokens": 101254996, "loss_tokens": 73099, "loss": 2.517730021956525, "lr": 0.00029982268319999716, "optimizer_lrs": [0.01998817887999981, 0.00029982268319999716], "gradient_norm": 0.42728036642074585, "seconds": 0.27420926600001394, "nonpadding_tokens_per_second": 266581.0716987087, "ranks": [{"compute_seconds": 0.22086785400000508, "nonpadding_tokens": 3082, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22365182099997583, "nonpadding_tokens": 12740, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22319327200000316, "nonpadding_tokens": 13197, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2213567090000197, "nonpadding_tokens": 8460, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21973534699998254, "nonpadding_tokens": 8247, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22106419199997163, "nonpadding_tokens": 10114, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.21979968499999814, "nonpadding_tokens": 6810, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22006133000002137, "nonpadding_tokens": 10449, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.011587238000004163, "prefetch": {"data_wait_seconds": 0.006684083999871859, "consumed_batches": 709, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1259, "tokens": 101332155, "loss_tokens": 77159, "loss": 2.3375695803470755, "lr": 0.0002998223463572019, "optimizer_lrs": [0.019988156423813463, 0.0002998223463572019], "gradient_norm": 0.3741638958454132, "seconds": 0.39786428200000046, "nonpadding_tokens_per_second": 193932.96531202545, "ranks": [{"compute_seconds": 0.3954865469999618, "nonpadding_tokens": 18688, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3954238609999834, "nonpadding_tokens": 2450, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39469747100002905, "nonpadding_tokens": 6358, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.392974009999989, "nonpadding_tokens": 8324, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.390763708999998, "nonpadding_tokens": 8131, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39443912400003, "nonpadding_tokens": 5439, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.39092171900000494, "nonpadding_tokens": 16907, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.39202711199999385, "nonpadding_tokens": 10862, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038422100004709137, "prefetch": {"data_wait_seconds": 0.006693248999866341, "consumed_batches": 710, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1260, "tokens": 101389533, "loss_tokens": 57378, "loss": 2.561122893356339, "lr": 0.0002998220956625998, "optimizer_lrs": [0.019988139710839986, 0.0002998220956625998], "gradient_norm": 0.4223983585834503, "seconds": 0.28290092599996797, "nonpadding_tokens_per_second": 202820.12085038738, "ranks": [{"compute_seconds": 0.278181907999965, "nonpadding_tokens": 2843, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.280178119000027, "nonpadding_tokens": 12515, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2771311020000553, "nonpadding_tokens": 9199, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2777176899999745, "nonpadding_tokens": 14213, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2801313930000333, "nonpadding_tokens": 4227, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2787796450000428, "nonpadding_tokens": 1731, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28000904299995, "nonpadding_tokens": 8258, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28058138400001553, "nonpadding_tokens": 4392, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034705199999507386, "prefetch": {"data_wait_seconds": 0.00670233099988593, "consumed_batches": 711, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1261, "tokens": 101466787, "loss_tokens": 77254, "loss": 2.344827614104124, "lr": 0.0002998217578472242, "optimizer_lrs": [0.019988117189814948, 0.0002998217578472242], "gradient_norm": 0.8137063980102539, "seconds": 0.3316514559999746, "nonpadding_tokens_per_second": 232937.31597549786, "ranks": [{"compute_seconds": 0.32925344400001677, "nonpadding_tokens": 17638, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32874576199998273, "nonpadding_tokens": 8068, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32536403400001745, "nonpadding_tokens": 5101, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3286313179999638, "nonpadding_tokens": 2311, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32461402699999553, "nonpadding_tokens": 2658, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32574937600003295, "nonpadding_tokens": 7742, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3244632079999974, "nonpadding_tokens": 26226, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3251206179999713, "nonpadding_tokens": 7510, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039860499998667365, "prefetch": {"data_wait_seconds": 0.006712731999883204, "consumed_batches": 712, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1262, "tokens": 101530318, "loss_tokens": 63531, "loss": 2.836620114589728, "lr": 0.0002998214797996746, "optimizer_lrs": [0.01998809865331164, 0.0002998214797996746], "gradient_norm": 0.4178909659385681, "seconds": 0.3159428100000241, "nonpadding_tokens_per_second": 201083.86071515648, "ranks": [{"compute_seconds": 0.31230541899998343, "nonpadding_tokens": 4547, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31110923599999296, "nonpadding_tokens": 7524, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3135807239999622, "nonpadding_tokens": 20117, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3119343949999802, "nonpadding_tokens": 5722, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3106245079999894, "nonpadding_tokens": 10761, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3134232049999923, "nonpadding_tokens": 3204, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3102416130000165, "nonpadding_tokens": 9013, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3110067559999834, "nonpadding_tokens": 2643, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003798729999857642, "prefetch": {"data_wait_seconds": 0.006722110999831443, "consumed_batches": 713, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1263, "tokens": 101620554, "loss_tokens": 90236, "loss": 2.684934851389689, "lr": 0.00029982108450372577, "optimizer_lrs": [0.019988072300248384, 0.00029982108450372577], "gradient_norm": 0.37154683470726013, "seconds": 0.28244434300000876, "nonpadding_tokens_per_second": 319482.4121508328, "ranks": [{"compute_seconds": 0.2794479629999955, "nonpadding_tokens": 14802, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2760456670000053, "nonpadding_tokens": 9053, "peak_allocated_bytes": 14985031168, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27723655499994493, "nonpadding_tokens": 10038, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2774663310000278, "nonpadding_tokens": 9482, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27555288099995323, "nonpadding_tokens": 6150, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2769525040000076, "nonpadding_tokens": 11478, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27566018100003475, "nonpadding_tokens": 4823, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2801707540000393, "nonpadding_tokens": 24410, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003464310000254045, "prefetch": {"data_wait_seconds": 0.00673123299982592, "consumed_batches": 714, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1264, "tokens": 101705777, "loss_tokens": 85223, "loss": 2.616344216056698, "lr": 0.00029982071076703124, "optimizer_lrs": [0.01998804738446875, 0.00029982071076703124], "gradient_norm": 0.42718803882598877, "seconds": 0.32956521900007374, "nonpadding_tokens_per_second": 258592.21509652367, "ranks": [{"compute_seconds": 0.3272940509999671, "nonpadding_tokens": 13099, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32714663500007646, "nonpadding_tokens": 17110, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3260735820000491, "nonpadding_tokens": 6216, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32473931300000913, "nonpadding_tokens": 14519, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32345180700008314, "nonpadding_tokens": 13550, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3257488629999443, "nonpadding_tokens": 4876, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.32696237300001485, "nonpadding_tokens": 10845, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3241832460000751, "nonpadding_tokens": 5008, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037785400002121605, "prefetch": {"data_wait_seconds": 0.006741246999879991, "consumed_batches": 715, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1265, "tokens": 101775479, "loss_tokens": 69702, "loss": 2.5323085241456487, "lr": 0.00029982040480639404, "optimizer_lrs": [0.019988026987092938, 0.00029982040480639404], "gradient_norm": 0.36302870512008667, "seconds": 0.32480397699998775, "nonpadding_tokens_per_second": 214597.1260690648, "ranks": [{"compute_seconds": 0.32035026899995955, "nonpadding_tokens": 18774, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31803582499992444, "nonpadding_tokens": 9588, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31868162800003574, "nonpadding_tokens": 7332, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3192516760000217, "nonpadding_tokens": 4891, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3219615509999585, "nonpadding_tokens": 6773, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32057615700000497, "nonpadding_tokens": 5182, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31760758900009023, "nonpadding_tokens": 4918, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32246420299998135, "nonpadding_tokens": 12244, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004019230000267271, "prefetch": {"data_wait_seconds": 0.006749406999873031, "consumed_batches": 716, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1266, "tokens": 101889055, "loss_tokens": 113576, "loss": 2.6285026105867435, "lr": 0.00029981990570007657, "optimizer_lrs": [0.01998799371333844, 0.00029981990570007657], "gradient_norm": 0.4642074406147003, "seconds": 0.2751713300000347, "nonpadding_tokens_per_second": 412746.48779720505, "ranks": [{"compute_seconds": 0.27042986799995106, "nonpadding_tokens": 3678, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2727644160000864, "nonpadding_tokens": 15906, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27158643999996457, "nonpadding_tokens": 15945, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2700866480000741, "nonpadding_tokens": 6600, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26871071100003974, "nonpadding_tokens": 19048, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2700205599999208, "nonpadding_tokens": 26415, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2681639649999852, "nonpadding_tokens": 21678, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2728723320000199, "nonpadding_tokens": 4306, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003918019999673561, "prefetch": {"data_wait_seconds": 0.006773143999865283, "consumed_batches": 717, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1267, "tokens": 101940602, "loss_tokens": 51547, "loss": 2.776713787902303, "lr": 0.00029981967894999773, "optimizer_lrs": [0.01998797859666652, 0.00029981967894999773], "gradient_norm": 0.4353136122226715, "seconds": 0.23462036900002659, "nonpadding_tokens_per_second": 219703.8569997056, "ranks": [{"compute_seconds": 0.23227316999998493, "nonpadding_tokens": 1781, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2310640540000577, "nonpadding_tokens": 5767, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2284012630000234, "nonpadding_tokens": 10212, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23161446599999636, "nonpadding_tokens": 3304, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2315608889999794, "nonpadding_tokens": 3687, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22841358499999842, "nonpadding_tokens": 13823, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.22704436300000452, "nonpadding_tokens": 3305, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22857241100007286, "nonpadding_tokens": 9668, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003786160000345262, "prefetch": {"data_wait_seconds": 0.00678212799989808, "consumed_batches": 718, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1268, "tokens": 102010021, "loss_tokens": 69419, "loss": 2.6356666582635877, "lr": 0.00029981937335751347, "optimizer_lrs": [0.019987958223834233, 0.00029981937335751347], "gradient_norm": 2.331786632537842, "seconds": 0.2503171519999796, "nonpadding_tokens_per_second": 277324.18432119925, "ranks": [{"compute_seconds": 0.24551810200000546, "nonpadding_tokens": 3570, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2479297449999649, "nonpadding_tokens": 2439, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24586826699999165, "nonpadding_tokens": 9548, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24618883000005098, "nonpadding_tokens": 4553, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24421827399999074, "nonpadding_tokens": 5006, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24705422399995314, "nonpadding_tokens": 17059, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24624735600002623, "nonpadding_tokens": 3355, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2446062090000396, "nonpadding_tokens": 23889, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005323630000475532, "prefetch": {"data_wait_seconds": 0.00679048299987528, "consumed_batches": 719, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1269, "tokens": 102047810, "loss_tokens": 37789, "loss": 2.735434336050173, "lr": 0.0002998192068961851, "optimizer_lrs": [0.01998794712641234, 0.0002998192068961851], "gradient_norm": 0.5002983808517456, "seconds": 0.28541885799995725, "nonpadding_tokens_per_second": 132398.3995479572, "ranks": [{"compute_seconds": 0.2813827989999709, "nonpadding_tokens": 6063, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2828803020000805, "nonpadding_tokens": 3888, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.280031235000024, "nonpadding_tokens": 1253, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28019366199998785, "nonpadding_tokens": 3434, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28300244199999725, "nonpadding_tokens": 4078, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.281561594999971, "nonpadding_tokens": 5857, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28245211000000836, "nonpadding_tokens": 10292, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27931231699994896, "nonpadding_tokens": 2924, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036368899998251436, "prefetch": {"data_wait_seconds": 0.006799855999886972, "consumed_batches": 720, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1270, "tokens": 102097997, "loss_tokens": 50187, "loss": 2.8473288027776116, "lr": 0.0002998189857029642, "optimizer_lrs": [0.019987932380197616, 0.0002998189857029642], "gradient_norm": 0.4688579738140106, "seconds": 0.23241904100007105, "nonpadding_tokens_per_second": 215933.2547972464, "ranks": [{"compute_seconds": 0.2296937699999262, "nonpadding_tokens": 7493, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2261473510000087, "nonpadding_tokens": 11839, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22668331099998795, "nonpadding_tokens": 3435, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22932674300000144, "nonpadding_tokens": 4925, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.22603783300007763, "nonpadding_tokens": 10695, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22888909999994667, "nonpadding_tokens": 4449, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.22609660299997358, "nonpadding_tokens": 3262, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23010304900003575, "nonpadding_tokens": 4089, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004024000000981687, "prefetch": {"data_wait_seconds": 0.006808267999829809, "consumed_batches": 721, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1271, "tokens": 102157602, "loss_tokens": 59605, "loss": 2.5847047751866454, "lr": 0.000299818722825475, "optimizer_lrs": [0.01998791485503167, 0.000299818722825475], "gradient_norm": 0.4893713891506195, "seconds": 0.27551369600007547, "nonpadding_tokens_per_second": 216341.33208384557, "ranks": [{"compute_seconds": 0.27032542699998885, "nonpadding_tokens": 9897, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2729453660000445, "nonpadding_tokens": 4455, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27196976499999437, "nonpadding_tokens": 5916, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27088076100005765, "nonpadding_tokens": 18396, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27284978799991677, "nonpadding_tokens": 5938, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27166008200003944, "nonpadding_tokens": 7439, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2685862549999456, "nonpadding_tokens": 3991, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27315270499991584, "nonpadding_tokens": 3573, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042250700005297404, "prefetch": {"data_wait_seconds": 0.006818656999897144, "consumed_batches": 722, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1272, "tokens": 102201307, "loss_tokens": 43705, "loss": 2.9636670932959617, "lr": 0.000299818529951046, "optimizer_lrs": [0.019987901996736404, 0.000299818529951046], "gradient_norm": 0.4303360879421234, "seconds": 0.2902521670000624, "nonpadding_tokens_per_second": 150575.9645198122, "ranks": [{"compute_seconds": 0.28673778199993194, "nonpadding_tokens": 5991, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2853664760000356, "nonpadding_tokens": 3451, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2879253629999994, "nonpadding_tokens": 8984, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28691377499990267, "nonpadding_tokens": 4938, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2851156930000798, "nonpadding_tokens": 2081, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28623012299999573, "nonpadding_tokens": 3527, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28466534200003935, "nonpadding_tokens": 11322, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2855440719999933, "nonpadding_tokens": 3411, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035074599998097256, "prefetch": {"data_wait_seconds": 0.0068279919998559535, "consumed_batches": 723, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1273, "tokens": 102267191, "loss_tokens": 65884, "loss": 2.6784077886892113, "lr": 0.0002998182390048361, "optimizer_lrs": [0.01998788260032241, 0.0002998182390048361], "gradient_norm": 0.4376595914363861, "seconds": 0.2648523389999582, "nonpadding_tokens_per_second": 248757.4784076587, "ranks": [{"compute_seconds": 0.2602076309999575, "nonpadding_tokens": 7454, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2588042210001049, "nonpadding_tokens": 4806, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26165718200002175, "nonpadding_tokens": 4710, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.261787647999995, "nonpadding_tokens": 3051, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25807233199998336, "nonpadding_tokens": 4471, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2610442170000624, "nonpadding_tokens": 10831, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.25808727399999043, "nonpadding_tokens": 10992, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2623981100000492, "nonpadding_tokens": 19569, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038939199998822005, "prefetch": {"data_wait_seconds": 0.00683629999987545, "consumed_batches": 724, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1274, "tokens": 102316992, "loss_tokens": 49801, "loss": 2.6194408433063594, "lr": 0.00029981801892720397, "optimizer_lrs": [0.019987867928480264, 0.00029981801892720397], "gradient_norm": 0.41953858733177185, "seconds": 0.25041398299993034, "nonpadding_tokens_per_second": 198874.67705832486, "ranks": [{"compute_seconds": 0.24816907499996432, "nonpadding_tokens": 4828, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2476920770000106, "nonpadding_tokens": 2012, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24490808200005176, "nonpadding_tokens": 3226, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2467468889999509, "nonpadding_tokens": 8175, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24750860100004957, "nonpadding_tokens": 10250, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2462616139999909, "nonpadding_tokens": 8506, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24315928199996506, "nonpadding_tokens": 9284, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24407228600000508, "nonpadding_tokens": 3520, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000394026999970265, "prefetch": {"data_wait_seconds": 0.006843379999807553, "consumed_batches": 725, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1275, "tokens": 102380047, "loss_tokens": 63055, "loss": 2.841119112282928, "lr": 0.0002998177400873806, "optimizer_lrs": [0.01998784933915871, 0.0002998177400873806], "gradient_norm": 0.4058283567428589, "seconds": 0.3255910789999916, "nonpadding_tokens_per_second": 193663.16851697778, "ranks": [{"compute_seconds": 0.24717334500007837, "nonpadding_tokens": 8317, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25124895400006153, "nonpadding_tokens": 9711, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24843801899999107, "nonpadding_tokens": 6522, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25256612399994083, "nonpadding_tokens": 6648, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24758736000001136, "nonpadding_tokens": 10372, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25011643400000594, "nonpadding_tokens": 5259, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24730536799995662, "nonpadding_tokens": 10574, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2536943559999827, "nonpadding_tokens": 5652, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.03142732100002377, "prefetch": {"data_wait_seconds": 0.0068517629998297025, "consumed_batches": 726, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1276, "tokens": 102448695, "loss_tokens": 68648, "loss": 2.9321705020539564, "lr": 0.0002998174362718309, "optimizer_lrs": [0.01998782908478873, 0.0002998174362718309], "gradient_norm": 0.42122551798820496, "seconds": 0.24471336999999949, "nonpadding_tokens_per_second": 280524.1086745695, "ranks": [{"compute_seconds": 0.24171113299996705, "nonpadding_tokens": 8425, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24208377500008282, "nonpadding_tokens": 4878, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24124453000001722, "nonpadding_tokens": 17346, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23978378799995426, "nonpadding_tokens": 16233, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24237452800002757, "nonpadding_tokens": 9493, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23984345100006976, "nonpadding_tokens": 2675, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23798251300001994, "nonpadding_tokens": 2434, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23929043399994043, "nonpadding_tokens": 7164, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039552199996251147, "prefetch": {"data_wait_seconds": 0.00686977299979219, "consumed_batches": 727, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1277, "tokens": 102523221, "loss_tokens": 74526, "loss": 2.626524635697609, "lr": 0.0002998171061557691, "optimizer_lrs": [0.019987807077051275, 0.0002998171061557691], "gradient_norm": 0.48769351840019226, "seconds": 0.27812195700005304, "nonpadding_tokens_per_second": 267961.58348614594, "ranks": [{"compute_seconds": 0.26721782999993593, "nonpadding_tokens": 14397, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26619526199999655, "nonpadding_tokens": 9851, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26603840699999637, "nonpadding_tokens": 13490, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26621173699993506, "nonpadding_tokens": 4449, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2647056099999645, "nonpadding_tokens": 5709, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2669065950000231, "nonpadding_tokens": 5400, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2694254610000826, "nonpadding_tokens": 11165, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2700136889999385, "nonpadding_tokens": 10065, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003560689999630995, "prefetch": {"data_wait_seconds": 0.0068784129998249455, "consumed_batches": 728, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1278, "tokens": 102576471, "loss_tokens": 53250, "loss": 2.7277705399061034, "lr": 0.000299816870100121, "optimizer_lrs": [0.019987791340008068, 0.000299816870100121], "gradient_norm": 0.47963014245033264, "seconds": 0.24742254700004196, "nonpadding_tokens_per_second": 215218.86604776955, "ranks": [{"compute_seconds": 0.24507155499998134, "nonpadding_tokens": 5482, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24146499799996946, "nonpadding_tokens": 4948, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24392948199999864, "nonpadding_tokens": 8318, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24268918999996458, "nonpadding_tokens": 8359, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2408247849999725, "nonpadding_tokens": 8710, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2436782319999793, "nonpadding_tokens": 6478, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24473465199992006, "nonpadding_tokens": 6369, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24160891400003948, "nonpadding_tokens": 4586, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037585099994430493, "prefetch": {"data_wait_seconds": 0.006886222999881397, "consumed_batches": 729, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1279, "tokens": 102643013, "loss_tokens": 66542, "loss": 2.7514927132487754, "lr": 0.0002998165749076017, "optimizer_lrs": [0.019987771660506783, 0.0002998165749076017], "gradient_norm": 0.4311327338218689, "seconds": 0.34529453599998305, "nonpadding_tokens_per_second": 192710.83976840938, "ranks": [{"compute_seconds": 0.34274034899999606, "nonpadding_tokens": 9553, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3424520539999776, "nonpadding_tokens": 4687, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33957336399998894, "nonpadding_tokens": 4196, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34194040199997744, "nonpadding_tokens": 14157, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33824260699998376, "nonpadding_tokens": 3635, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3398158899999544, "nonpadding_tokens": 3732, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.342187019999983, "nonpadding_tokens": 6377, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3391366259999131, "nonpadding_tokens": 20205, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003549549999206647, "prefetch": {"data_wait_seconds": 0.0068952299998983335, "consumed_batches": 730, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1280, "tokens": 102696252, "loss_tokens": 53239, "loss": 2.548281980784763, "lr": 0.0002998163385585615, "optimizer_lrs": [0.0199877559039041, 0.0002998163385585615], "gradient_norm": 0.49029994010925293, "seconds": 0.3276256230000172, "nonpadding_tokens_per_second": 162499.50022986208, "ranks": [{"compute_seconds": 0.32402192099993954, "nonpadding_tokens": 3341, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3251917179999282, "nonpadding_tokens": 17219, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32376237000005403, "nonpadding_tokens": 14058, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3242233140000508, "nonpadding_tokens": 8787, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32080439099991054, "nonpadding_tokens": 3627, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32420343700005105, "nonpadding_tokens": 1872, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.32449568499998804, "nonpadding_tokens": 1961, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3253475609999441, "nonpadding_tokens": 2374, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003829640000958534, "prefetch": {"data_wait_seconds": 0.006904185999886181, "consumed_batches": 731, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1281, "tokens": 102761908, "loss_tokens": 65656, "loss": 2.978067503350798, "lr": 0.00029981604687612544, "optimizer_lrs": [0.019987736458408362, 0.00029981604687612544], "gradient_norm": 0.3834269642829895, "seconds": 0.2773013589999209, "nonpadding_tokens_per_second": 236767.68205098744, "ranks": [{"compute_seconds": 0.2506175500000154, "nonpadding_tokens": 2129, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26181510399999297, "nonpadding_tokens": 9664, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.258299518000058, "nonpadding_tokens": 19115, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2648802999999589, "nonpadding_tokens": 3804, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25745253200000207, "nonpadding_tokens": 2263, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26039981400003853, "nonpadding_tokens": 12679, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.25743763499997385, "nonpadding_tokens": 11600, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26633209600004193, "nonpadding_tokens": 4402, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004496540000218374, "prefetch": {"data_wait_seconds": 0.00691270399995858, "consumed_batches": 732, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1282, "tokens": 102820214, "loss_tokens": 58306, "loss": 2.498595502606936, "lr": 0.000299815787652808, "optimizer_lrs": [0.019987719176853866, 0.000299815787652808], "gradient_norm": 0.3578074872493744, "seconds": 0.26013073199999326, "nonpadding_tokens_per_second": 224141.14453805293, "ranks": [{"compute_seconds": 0.2538200300000426, "nonpadding_tokens": 7851, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2538877810000031, "nonpadding_tokens": 6179, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25249381200001153, "nonpadding_tokens": 16710, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2567649129999836, "nonpadding_tokens": 1957, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25740077099999326, "nonpadding_tokens": 3363, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.253910602000019, "nonpadding_tokens": 6733, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.25142271700008223, "nonpadding_tokens": 3701, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2575765550000142, "nonpadding_tokens": 11812, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003982530000712359, "prefetch": {"data_wait_seconds": 0.006922189999912121, "consumed_batches": 733, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1283, "tokens": 102887087, "loss_tokens": 66873, "loss": 2.550683992418465, "lr": 0.00029981549011680564, "optimizer_lrs": [0.019987699341120376, 0.00029981549011680564], "gradient_norm": 0.3734212815761566, "seconds": 0.260247374999949, "nonpadding_tokens_per_second": 256959.36414349274, "ranks": [{"compute_seconds": 0.2580931300000202, "nonpadding_tokens": 3873, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2552618930000108, "nonpadding_tokens": 5635, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25213156600000275, "nonpadding_tokens": 9857, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2526928359999374, "nonpadding_tokens": 7429, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25110086899996986, "nonpadding_tokens": 13959, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2542934680000144, "nonpadding_tokens": 16722, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2577730140000085, "nonpadding_tokens": 5887, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2518416460000026, "nonpadding_tokens": 3511, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033681100001103914, "prefetch": {"data_wait_seconds": 0.0069303339999606806, "consumed_batches": 734, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1284, "tokens": 102973997, "loss_tokens": 86910, "loss": 2.2237296484869407, "lr": 0.0002998151030722566, "optimizer_lrs": [0.019987673538150442, 0.0002998151030722566], "gradient_norm": 0.5992882251739502, "seconds": 0.32412804900002357, "nonpadding_tokens_per_second": 268134.7704036366, "ranks": [{"compute_seconds": 0.321832198999914, "nonpadding_tokens": 15221, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32131138299996564, "nonpadding_tokens": 5748, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3207045660000176, "nonpadding_tokens": 9135, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3189637110000376, "nonpadding_tokens": 10626, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3217664530000093, "nonpadding_tokens": 14360, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3191714700000148, "nonpadding_tokens": 17674, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3175112440000021, "nonpadding_tokens": 5763, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3191166259999818, "nonpadding_tokens": 8383, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033913100003246655, "prefetch": {"data_wait_seconds": 0.006937794999956282, "consumed_batches": 735, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1285, "tokens": 103035030, "loss_tokens": 61033, "loss": 2.6133633649009553, "lr": 0.0002998148310259875, "optimizer_lrs": [0.0199876554017325, 0.0002998148310259875], "gradient_norm": 0.9183235764503479, "seconds": 0.2788757100000794, "nonpadding_tokens_per_second": 218853.76822521628, "ranks": [{"compute_seconds": 0.2765519099999665, "nonpadding_tokens": 8724, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27322838500003854, "nonpadding_tokens": 11297, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27561755800002175, "nonpadding_tokens": 2972, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.274298994999981, "nonpadding_tokens": 4519, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2733197449999807, "nonpadding_tokens": 13665, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27409861599994656, "nonpadding_tokens": 11116, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27265112600002794, "nonpadding_tokens": 4859, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27317629499998475, "nonpadding_tokens": 3881, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036105399999541987, "prefetch": {"data_wait_seconds": 0.006947210999854292, "consumed_batches": 736, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1286, "tokens": 103096943, "loss_tokens": 61913, "loss": 2.694671605720931, "lr": 0.0002998145548530436, "optimizer_lrs": [0.019987636990202905, 0.0002998145548530436], "gradient_norm": 0.6661251187324524, "seconds": 0.29808118200003264, "nonpadding_tokens_per_second": 207705.16134089007, "ranks": [{"compute_seconds": 0.2953376530000469, "nonpadding_tokens": 5440, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2958266179999782, "nonpadding_tokens": 11735, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29271035600004325, "nonpadding_tokens": 6943, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2931998429999112, "nonpadding_tokens": 4791, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29573753600004693, "nonpadding_tokens": 11125, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29416197900002317, "nonpadding_tokens": 6310, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2952916729999515, "nonpadding_tokens": 11404, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29563911000002463, "nonpadding_tokens": 4165, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000398373000052743, "prefetch": {"data_wait_seconds": 0.006956259999881809, "consumed_batches": 737, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1287, "tokens": 103154904, "loss_tokens": 57961, "loss": 2.6663227752281706, "lr": 0.0002998142961222504, "optimizer_lrs": [0.019987619741483362, 0.0002998142961222504], "gradient_norm": 0.5552670359611511, "seconds": 0.2976031740000735, "nonpadding_tokens_per_second": 194759.34755986737, "ranks": [{"compute_seconds": 0.2943401320000021, "nonpadding_tokens": 3320, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2914736500000572, "nonpadding_tokens": 2962, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2915494389999367, "nonpadding_tokens": 3114, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2925600649999751, "nonpadding_tokens": 1911, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2952418630000011, "nonpadding_tokens": 9653, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2935174519999464, "nonpadding_tokens": 17052, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.29068818700000065, "nonpadding_tokens": 10644, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29129667199993037, "nonpadding_tokens": 9305, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000371518999941145, "prefetch": {"data_wait_seconds": 0.006965765999964901, "consumed_batches": 738, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1288, "tokens": 103205390, "loss_tokens": 50486, "loss": 2.7628714519866895, "lr": 0.0002998140706120693, "optimizer_lrs": [0.019987604707471286, 0.0002998140706120693], "gradient_norm": 0.4054900109767914, "seconds": 0.30320245100006105, "nonpadding_tokens_per_second": 166509.2080670213, "ranks": [{"compute_seconds": 0.30063954499996726, "nonpadding_tokens": 6363, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30088052899998274, "nonpadding_tokens": 7665, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2978978889999553, "nonpadding_tokens": 3624, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3007570700000315, "nonpadding_tokens": 2241, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29687282000008963, "nonpadding_tokens": 13695, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29970504000004894, "nonpadding_tokens": 8764, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.29664659900004153, "nonpadding_tokens": 3922, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29724922299999434, "nonpadding_tokens": 4212, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003587850000030812, "prefetch": {"data_wait_seconds": 0.006975782999987246, "consumed_batches": 739, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1289, "tokens": 103263598, "loss_tokens": 58208, "loss": 2.5021297545010994, "lr": 0.0002998138104396426, "optimizer_lrs": [0.01998758736264284, 0.0002998138104396426], "gradient_norm": 0.9880985617637634, "seconds": 0.3200784809999959, "nonpadding_tokens_per_second": 181855.39939500258, "ranks": [{"compute_seconds": 0.3177503760000491, "nonpadding_tokens": 20373, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3171746450000228, "nonpadding_tokens": 11316, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31590034299995295, "nonpadding_tokens": 4602, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3145799310000257, "nonpadding_tokens": 6114, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3169784649999201, "nonpadding_tokens": 3357, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3144875809999803, "nonpadding_tokens": 3862, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3130613360000325, "nonpadding_tokens": 3892, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3138149469999689, "nonpadding_tokens": 4692, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037412300002870325, "prefetch": {"data_wait_seconds": 0.006983812999990846, "consumed_batches": 740, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1290, "tokens": 103327824, "loss_tokens": 64226, "loss": 2.7640811353657395, "lr": 0.0002998135231576083, "optimizer_lrs": [0.01998756821050722, 0.0002998135231576083], "gradient_norm": 0.40275704860687256, "seconds": 0.269100293000065, "nonpadding_tokens_per_second": 238669.3796724498, "ranks": [{"compute_seconds": 0.26473009200003617, "nonpadding_tokens": 3501, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2651240970000117, "nonpadding_tokens": 9762, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26517800600004193, "nonpadding_tokens": 4468, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26304176199994345, "nonpadding_tokens": 17466, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.261486360000049, "nonpadding_tokens": 6031, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2643226220000088, "nonpadding_tokens": 7323, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2611757150000358, "nonpadding_tokens": 12165, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2667089889999943, "nonpadding_tokens": 3510, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036115399996106134, "prefetch": {"data_wait_seconds": 0.006992319999952201, "consumed_batches": 741, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1291, "tokens": 103400649, "loss_tokens": 72825, "loss": 2.597077754891864, "lr": 0.0002998131971446263, "optimizer_lrs": [0.01998754647630842, 0.0002998131971446263], "gradient_norm": 0.49470916390419006, "seconds": 0.27928701099995124, "nonpadding_tokens_per_second": 260753.26503463034, "ranks": [{"compute_seconds": 0.27448757899992415, "nonpadding_tokens": 6552, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27312528400000247, "nonpadding_tokens": 2782, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27575373399997716, "nonpadding_tokens": 3260, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2741252660000555, "nonpadding_tokens": 13684, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27689540400001533, "nonpadding_tokens": 7147, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2743068619998894, "nonpadding_tokens": 17296, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2765559160000066, "nonpadding_tokens": 16739, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27354115200000706, "nonpadding_tokens": 5365, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045392999993509875, "prefetch": {"data_wait_seconds": 0.007003749000048742, "consumed_batches": 742, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1292, "tokens": 103473827, "loss_tokens": 73178, "loss": 2.5825109237065784, "lr": 0.000299812869264776, "optimizer_lrs": [0.019987524617651735, 0.000299812869264776], "gradient_norm": 0.6331342458724976, "seconds": 0.34366921499997716, "nonpadding_tokens_per_second": 212931.49576986366, "ranks": [{"compute_seconds": 0.33579140500000904, "nonpadding_tokens": 9275, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34064044199999444, "nonpadding_tokens": 17896, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.339943622000078, "nonpadding_tokens": 6763, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3384027470000319, "nonpadding_tokens": 8436, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3412798479999992, "nonpadding_tokens": 4578, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3397744649998913, "nonpadding_tokens": 17645, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3408569670000361, "nonpadding_tokens": 3933, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3374793849999378, "nonpadding_tokens": 4652, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039550999997572944, "prefetch": {"data_wait_seconds": 0.007012878000026035, "consumed_batches": 743, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1293, "tokens": 103523280, "loss_tokens": 49453, "loss": 2.686747391462601, "lr": 0.00029981264752399503, "optimizer_lrs": [0.019987509834933, 0.00029981264752399503], "gradient_norm": 0.4741740822792053, "seconds": 0.34908067000003484, "nonpadding_tokens_per_second": 141666.39476197597, "ranks": [{"compute_seconds": 0.34508753199997955, "nonpadding_tokens": 10514, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3465719910000189, "nonpadding_tokens": 4122, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34336484099992504, "nonpadding_tokens": 3757, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3440284650000649, "nonpadding_tokens": 2616, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3443181469999672, "nonpadding_tokens": 4538, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34132460600005743, "nonpadding_tokens": 2528, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3422804439999254, "nonpadding_tokens": 10458, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3429194389999566, "nonpadding_tokens": 10920, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034667299996726797, "prefetch": {"data_wait_seconds": 0.007020942000011132, "consumed_batches": 744, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1294, "tokens": 103591969, "loss_tokens": 68689, "loss": 2.9175027114967462, "lr": 0.0002998123393138198, "optimizer_lrs": [0.01998748928758799, 0.0002998123393138198], "gradient_norm": 0.4111570715904236, "seconds": 0.37297346100001505, "nonpadding_tokens_per_second": 184165.91844318176, "ranks": [{"compute_seconds": 0.3705103660000759, "nonpadding_tokens": 18408, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37000243500006036, "nonpadding_tokens": 6967, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36901590299999043, "nonpadding_tokens": 10753, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3674584430000323, "nonpadding_tokens": 1496, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.36978944899999533, "nonpadding_tokens": 7387, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36878374899993105, "nonpadding_tokens": 9544, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.36628839499996957, "nonpadding_tokens": 6669, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37030478700000913, "nonpadding_tokens": 7465, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004285129999743731, "prefetch": {"data_wait_seconds": 0.007030220000004306, "consumed_batches": 745, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1295, "tokens": 103663572, "loss_tokens": 71603, "loss": 2.472143930771057, "lr": 0.00029981201775894844, "optimizer_lrs": [0.019987467850596564, 0.00029981201775894844], "gradient_norm": 0.3770521879196167, "seconds": 0.2760240520000252, "nonpadding_tokens_per_second": 259408.5532806882, "ranks": [{"compute_seconds": 0.27349184299998797, "nonpadding_tokens": 7351, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27062239400004273, "nonpadding_tokens": 3779, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27205189500000415, "nonpadding_tokens": 15841, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27276968700005, "nonpadding_tokens": 8014, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2665568659999735, "nonpadding_tokens": 13756, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2705959800000528, "nonpadding_tokens": 9231, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2729218399999809, "nonpadding_tokens": 10535, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2735926440000185, "nonpadding_tokens": 3096, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036768200004644314, "prefetch": {"data_wait_seconds": 0.007038787000013258, "consumed_batches": 746, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1296, "tokens": 103724016, "loss_tokens": 60444, "loss": 2.7105065742670904, "lr": 0.0002998117461028304, "optimizer_lrs": [0.019987449740188692, 0.0002998117461028304], "gradient_norm": 0.3695668876171112, "seconds": 0.34022693499991874, "nonpadding_tokens_per_second": 177657.89178336054, "ranks": [{"compute_seconds": 0.33513605500002086, "nonpadding_tokens": 15718, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3372198389999994, "nonpadding_tokens": 9100, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33451018800008114, "nonpadding_tokens": 6409, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33691944999998213, "nonpadding_tokens": 3010, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3328507429999945, "nonpadding_tokens": 8411, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3347535310000467, "nonpadding_tokens": 3454, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3330190249999987, "nonpadding_tokens": 9084, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33782268200002363, "nonpadding_tokens": 5258, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004281180000589302, "prefetch": {"data_wait_seconds": 0.007052483999999026, "consumed_batches": 747, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1297, "tokens": 103814092, "loss_tokens": 90076, "loss": 2.8152787299058573, "lr": 0.0002998113409066173, "optimizer_lrs": [0.019987422727107823, 0.0002998113409066173], "gradient_norm": 0.44420361518859863, "seconds": 0.3164780949999795, "nonpadding_tokens_per_second": 284620.0145384654, "ranks": [{"compute_seconds": 0.3125795810000227, "nonpadding_tokens": 13639, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3100000250000221, "nonpadding_tokens": 13713, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3129099400000541, "nonpadding_tokens": 5287, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31338223800003107, "nonpadding_tokens": 10864, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.30941601200004243, "nonpadding_tokens": 4332, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3109944060000771, "nonpadding_tokens": 6058, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3135307529998954, "nonpadding_tokens": 20198, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31412473199998203, "nonpadding_tokens": 15985, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036044900002707436, "prefetch": {"data_wait_seconds": 0.007061573000058274, "consumed_batches": 748, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1298, "tokens": 103901240, "loss_tokens": 87148, "loss": 2.4574425474480193, "lr": 0.00029981094846736294, "optimizer_lrs": [0.019987396564490865, 0.00029981094846736294], "gradient_norm": 1.1058709621429443, "seconds": 0.3266630639999448, "nonpadding_tokens_per_second": 266782.5340670126, "ranks": [{"compute_seconds": 0.3236232149999978, "nonpadding_tokens": 3320, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32410980199995265, "nonpadding_tokens": 3403, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3234068950000619, "nonpadding_tokens": 11158, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3238116140000784, "nonpadding_tokens": 4510, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3198642520000021, "nonpadding_tokens": 4634, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32284942800004046, "nonpadding_tokens": 18220, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3240029199999981, "nonpadding_tokens": 15593, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.321107684000026, "nonpadding_tokens": 26310, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0006071380000776117, "prefetch": {"data_wait_seconds": 0.007071853000027772, "consumed_batches": 749, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1299, "tokens": 103976165, "loss_tokens": 74925, "loss": 2.624202118785452, "lr": 0.000299810610744192, "optimizer_lrs": [0.019987374049612802, 0.000299810610744192], "gradient_norm": 0.5769463777542114, "seconds": 0.3143341640000017, "nonpadding_tokens_per_second": 238360.981977128, "ranks": [{"compute_seconds": 0.3120771460000924, "nonpadding_tokens": 16199, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3112336620000633, "nonpadding_tokens": 3376, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3081517610000901, "nonpadding_tokens": 15289, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3107841760000838, "nonpadding_tokens": 7937, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3068877350000321, "nonpadding_tokens": 6102, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.308449937999967, "nonpadding_tokens": 10175, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3071192319999909, "nonpadding_tokens": 10734, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30800783500001216, "nonpadding_tokens": 5113, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035651799998959177, "prefetch": {"data_wait_seconds": 0.007081067000015651, "consumed_batches": 750, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1300, "tokens": 104037926, "loss_tokens": 61761, "loss": 2.499327042955911, "lr": 0.0002998103321310779, "optimizer_lrs": [0.019987355475405196, 0.0002998103321310779], "gradient_norm": 0.3944375813007355, "seconds": 0.29135378900002706, "nonpadding_tokens_per_second": 211979.39526365406, "ranks": [{"compute_seconds": 0.287064561999955, "nonpadding_tokens": 7123, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28850366000006034, "nonpadding_tokens": 8101, "peak_allocated_bytes": 14985223680, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2878725589999931, "nonpadding_tokens": 4424, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28689510899994275, "nonpadding_tokens": 5154, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2889885109999568, "nonpadding_tokens": 10371, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2864647110000078, "nonpadding_tokens": 9826, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28890547899993635, "nonpadding_tokens": 8502, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28553832399995827, "nonpadding_tokens": 8260, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004235139999764215, "prefetch": {"data_wait_seconds": 0.0070909620000776385, "consumed_batches": 751, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1301, "tokens": 104107845, "loss_tokens": 69919, "loss": 2.6126162505899684, "lr": 0.0002998100164690321, "optimizer_lrs": [0.019987334431268808, 0.0002998100164690321], "gradient_norm": 0.40550997853279114, "seconds": 0.29302908700003627, "nonpadding_tokens_per_second": 238607.71200502478, "ranks": [{"compute_seconds": 0.28853498099999797, "nonpadding_tokens": 6538, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2867690479999965, "nonpadding_tokens": 15032, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28734007699995345, "nonpadding_tokens": 16091, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28740670299998783, "nonpadding_tokens": 11754, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2902398449999737, "nonpadding_tokens": 6832, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2890568269999676, "nonpadding_tokens": 2048, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2902630149999368, "nonpadding_tokens": 5066, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2907334069999479, "nonpadding_tokens": 6558, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003406250000352884, "prefetch": {"data_wait_seconds": 0.0070993730001447375, "consumed_batches": 752, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1302, "tokens": 104165333, "loss_tokens": 57488, "loss": 2.651441115972029, "lr": 0.0002998097567325239, "optimizer_lrs": [0.019987317115501595, 0.0002998097567325239], "gradient_norm": 0.42966702580451965, "seconds": 0.3028816949999964, "nonpadding_tokens_per_second": 189803.480860739, "ranks": [{"compute_seconds": 0.30060258999992584, "nonpadding_tokens": 8947, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29682628299997305, "nonpadding_tokens": 8024, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2993971649999594, "nonpadding_tokens": 3248, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3004159230000596, "nonpadding_tokens": 10941, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.300515191000045, "nonpadding_tokens": 11587, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2991871980000269, "nonpadding_tokens": 3466, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3003563469999335, "nonpadding_tokens": 3841, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2971742510000013, "nonpadding_tokens": 7434, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003500110000231871, "prefetch": {"data_wait_seconds": 0.007107916000180126, "consumed_batches": 753, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1303, "tokens": 104216493, "loss_tokens": 51160, "loss": 2.5538346303264268, "lr": 0.00029980952543744735, "optimizer_lrs": [0.019987301695829823, 0.00029980952543744735], "gradient_norm": 0.4300478994846344, "seconds": 0.24227280599995993, "nonpadding_tokens_per_second": 211166.9107427949, "ranks": [{"compute_seconds": 0.23933837399999902, "nonpadding_tokens": 2561, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23672910000004777, "nonpadding_tokens": 9124, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23658874399995966, "nonpadding_tokens": 4896, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2399254669999209, "nonpadding_tokens": 6114, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23524483200003488, "nonpadding_tokens": 5120, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23798786200006816, "nonpadding_tokens": 10040, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23549190200003522, "nonpadding_tokens": 8325, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2356024680000246, "nonpadding_tokens": 4980, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043173500000648346, "prefetch": {"data_wait_seconds": 0.007125106000174242, "consumed_batches": 754, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1304, "tokens": 104283892, "loss_tokens": 67399, "loss": 2.647801015222778, "lr": 0.0002998092205112764, "optimizer_lrs": [0.019987281367418428, 0.0002998092205112764], "gradient_norm": 0.6868286728858948, "seconds": 0.2987155219999522, "nonpadding_tokens_per_second": 225629.38661088652, "ranks": [{"compute_seconds": 0.2959353229999806, "nonpadding_tokens": 9271, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2919169189999593, "nonpadding_tokens": 5452, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29517211200004567, "nonpadding_tokens": 6982, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29516726299993934, "nonpadding_tokens": 2349, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29146531699996103, "nonpadding_tokens": 2948, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2929273590000321, "nonpadding_tokens": 21137, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.29176387200004683, "nonpadding_tokens": 5615, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2924331250000023, "nonpadding_tokens": 13645, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005760020000025179, "prefetch": {"data_wait_seconds": 0.007131677000145942, "consumed_batches": 755, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1305, "tokens": 104353977, "loss_tokens": 70085, "loss": 2.7824110901048726, "lr": 0.00029980890317465083, "optimizer_lrs": [0.01998726021164339, 0.00029980890317465083], "gradient_norm": 0.43193942308425903, "seconds": 0.2601966900000434, "nonpadding_tokens_per_second": 269353.926062581, "ranks": [{"compute_seconds": 0.2567682810000633, "nonpadding_tokens": 3826, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25763481600006344, "nonpadding_tokens": 11977, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2541022210000392, "nonpadding_tokens": 10631, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2551393899999539, "nonpadding_tokens": 5915, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2531020650000073, "nonpadding_tokens": 18667, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25443790299993907, "nonpadding_tokens": 10543, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2529958779999788, "nonpadding_tokens": 6438, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2537072129999842, "nonpadding_tokens": 2088, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003502970000681671, "prefetch": {"data_wait_seconds": 0.00713956000011251, "consumed_batches": 756, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1306, "tokens": 104427370, "loss_tokens": 73393, "loss": 2.6878480832640714, "lr": 0.00029980857057732273, "optimizer_lrs": [0.019987238038488182, 0.00029980857057732273], "gradient_norm": 0.38392430543899536, "seconds": 0.3161533119999831, "nonpadding_tokens_per_second": 232143.70121798353, "ranks": [{"compute_seconds": 0.3132976830000871, "nonpadding_tokens": 6310, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31087463900007606, "nonpadding_tokens": 6369, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3107215869999891, "nonpadding_tokens": 15309, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3081777830000192, "nonpadding_tokens": 8699, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3117261029999554, "nonpadding_tokens": 5309, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3115407469999809, "nonpadding_tokens": 9418, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3135009249999712, "nonpadding_tokens": 10275, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.307031631999962, "nonpadding_tokens": 11704, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038218800000322517, "prefetch": {"data_wait_seconds": 0.007147728000134634, "consumed_batches": 757, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1307, "tokens": 104497563, "loss_tokens": 70193, "loss": 2.6947351854885815, "lr": 0.0002998082522111785, "optimizer_lrs": [0.01998721681407857, 0.0002998082522111785], "gradient_norm": 0.3788534104824066, "seconds": 0.41562125699999797, "nonpadding_tokens_per_second": 168886.93448131395, "ranks": [{"compute_seconds": 0.40696082199997363, "nonpadding_tokens": 6824, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4088793280000118, "nonpadding_tokens": 5146, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4107586900000797, "nonpadding_tokens": 1985, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4112773379999908, "nonpadding_tokens": 12506, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4052087899999606, "nonpadding_tokens": 9732, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4092487619999474, "nonpadding_tokens": 3595, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.41139010199992754, "nonpadding_tokens": 14879, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4130868529999816, "nonpadding_tokens": 15526, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043854900002315844, "prefetch": {"data_wait_seconds": 0.0071556340000853424, "consumed_batches": 758, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1308, "tokens": 104559059, "loss_tokens": 61496, "loss": 2.8372838782197216, "lr": 0.00029980797307376346, "optimizer_lrs": [0.019987198204917567, 0.00029980797307376346], "gradient_norm": 1.4151744842529297, "seconds": 0.2453039130000434, "nonpadding_tokens_per_second": 250693.10655468068, "ranks": [{"compute_seconds": 0.24275911500001257, "nonpadding_tokens": 4302, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23784116499996344, "nonpadding_tokens": 3479, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.235037315999989, "nonpadding_tokens": 7248, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24142640499997015, "nonpadding_tokens": 2923, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24219889200003308, "nonpadding_tokens": 4362, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23893064399999275, "nonpadding_tokens": 14210, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23428159499997037, "nonpadding_tokens": 13115, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23460755000007794, "nonpadding_tokens": 11857, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003436599999986356, "prefetch": {"data_wait_seconds": 0.0071652160001605125, "consumed_batches": 759, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1309, "tokens": 104606573, "loss_tokens": 47514, "loss": 2.3356022369722607, "lr": 0.00029980775726331816, "optimizer_lrs": [0.019987183817554544, 0.00029980775726331816], "gradient_norm": 0.4219188988208771, "seconds": 0.22591736200001833, "nonpadding_tokens_per_second": 210315.84106402652, "ranks": [{"compute_seconds": 0.21895370400000047, "nonpadding_tokens": 4721, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21984141699999782, "nonpadding_tokens": 8148, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2172971029999644, "nonpadding_tokens": 4476, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.222473339999965, "nonpadding_tokens": 3352, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.22300397500009694, "nonpadding_tokens": 2011, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2201275480000504, "nonpadding_tokens": 6589, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.22208389200000056, "nonpadding_tokens": 6309, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22304799500000172, "nonpadding_tokens": 11908, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004802419999805352, "prefetch": {"data_wait_seconds": 0.0071762270002437845, "consumed_batches": 760, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1310, "tokens": 104675706, "loss_tokens": 69133, "loss": 2.597540248506502, "lr": 0.0002998074430422157, "optimizer_lrs": [0.019987162869481045, 0.0002998074430422157], "gradient_norm": 0.48088809847831726, "seconds": 0.2841587279999658, "nonpadding_tokens_per_second": 243290.08116903002, "ranks": [{"compute_seconds": 0.2805216909999899, "nonpadding_tokens": 9953, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2793029800000113, "nonpadding_tokens": 10153, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2765296939999189, "nonpadding_tokens": 8823, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27714591699998437, "nonpadding_tokens": 4329, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2753429619999679, "nonpadding_tokens": 9248, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.278471829999944, "nonpadding_tokens": 6700, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2755160120000255, "nonpadding_tokens": 8466, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28133271899991996, "nonpadding_tokens": 11461, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040915400006724667, "prefetch": {"data_wait_seconds": 0.00718645800027673, "consumed_batches": 761, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1311, "tokens": 104743572, "loss_tokens": 67866, "loss": 2.699701203474494, "lr": 0.0002998071343304422, "optimizer_lrs": [0.019987142288696146, 0.0002998071343304422], "gradient_norm": 0.48142585158348083, "seconds": 0.2809809390000737, "nonpadding_tokens_per_second": 241532.39803922144, "ranks": [{"compute_seconds": 0.27812388200004534, "nonpadding_tokens": 5526, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27426921300002505, "nonpadding_tokens": 1795, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27652145700005804, "nonpadding_tokens": 2403, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2777954899999031, "nonpadding_tokens": 6363, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2779363330000706, "nonpadding_tokens": 16081, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2739471300000105, "nonpadding_tokens": 9638, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2699214099999381, "nonpadding_tokens": 15151, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27109776700001476, "nonpadding_tokens": 10909, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005904600000121718, "prefetch": {"data_wait_seconds": 0.007193712000301389, "consumed_batches": 762, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1312, "tokens": 104876710, "loss_tokens": 133138, "loss": 2.85071105732398, "lr": 0.0002998065279887493, "optimizer_lrs": [0.01998710186591662, 0.0002998065279887493], "gradient_norm": 0.4710376262664795, "seconds": 0.3442600670000502, "nonpadding_tokens_per_second": 386736.69345443015, "ranks": [{"compute_seconds": 0.33928203599998596, "nonpadding_tokens": 13604, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33789128300008997, "nonpadding_tokens": 21781, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3350577469999507, "nonpadding_tokens": 25981, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3361150399999815, "nonpadding_tokens": 12287, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34065060800003266, "nonpadding_tokens": 10178, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3369345560000738, "nonpadding_tokens": 21183, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.33429725500002405, "nonpadding_tokens": 10323, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34149069500006135, "nonpadding_tokens": 17801, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004652269999496639, "prefetch": {"data_wait_seconds": 0.007203861000292022, "consumed_batches": 763, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1313, "tokens": 104923997, "loss_tokens": 47287, "loss": 2.937615980607778, "lr": 0.0002998063124038001, "optimizer_lrs": [0.019987087493586676, 0.0002998063124038001], "gradient_norm": 0.47087904810905457, "seconds": 0.3434043299999985, "nonpadding_tokens_per_second": 137700.65159050326, "ranks": [{"compute_seconds": 0.33742754499996863, "nonpadding_tokens": 15373, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33656117800001084, "nonpadding_tokens": 3134, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33933883099996365, "nonpadding_tokens": 6778, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34056285399992703, "nonpadding_tokens": 5894, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34011773700001413, "nonpadding_tokens": 3192, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33795872899997903, "nonpadding_tokens": 3597, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3320695119999755, "nonpadding_tokens": 4557, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33385269299992615, "nonpadding_tokens": 4762, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046863699992627517, "prefetch": {"data_wait_seconds": 0.0072139310003223045, "consumed_batches": 764, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1314, "tokens": 104974449, "loss_tokens": 50452, "loss": 2.724699962340442, "lr": 0.00029980608225710874, "optimizer_lrs": [0.019987072150473916, 0.00029980608225710874], "gradient_norm": 0.46933943033218384, "seconds": 0.3579885840000543, "nonpadding_tokens_per_second": 140931.86837486512, "ranks": [{"compute_seconds": 0.3527858009999818, "nonpadding_tokens": 8497, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35065754699996887, "nonpadding_tokens": 1717, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35350481600005423, "nonpadding_tokens": 6177, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.348876851, "nonpadding_tokens": 4199, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3471813150000571, "nonpadding_tokens": 5161, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35012813699995604, "nonpadding_tokens": 5282, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3541396470000109, "nonpadding_tokens": 6978, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3552057250000189, "nonpadding_tokens": 12441, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046858400003202405, "prefetch": {"data_wait_seconds": 0.007240736000269976, "consumed_batches": 765, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1315, "tokens": 105018527, "loss_tokens": 44078, "loss": 2.6942508167339714, "lr": 0.0002998058810749016, "optimizer_lrs": [0.019987058738326777, 0.0002998058810749016], "gradient_norm": 0.41244956851005554, "seconds": 0.3489740539999957, "nonpadding_tokens_per_second": 126307.38444526464, "ranks": [{"compute_seconds": 0.34595899900000404, "nonpadding_tokens": 9823, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3425099670000691, "nonpadding_tokens": 4228, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3406891770000584, "nonpadding_tokens": 6270, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3413178580000249, "nonpadding_tokens": 5163, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3400414230000024, "nonpadding_tokens": 5046, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3427479880000419, "nonpadding_tokens": 10214, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3463675330000342, "nonpadding_tokens": 2082, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34046621400000276, "nonpadding_tokens": 1252, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003888070000357402, "prefetch": {"data_wait_seconds": 0.007250735000255304, "consumed_batches": 766, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1316, "tokens": 105081673, "loss_tokens": 63146, "loss": 2.7718090813353182, "lr": 0.000299805592680306, "optimizer_lrs": [0.0199870395120204, 0.000299805592680306], "gradient_norm": 0.5030546188354492, "seconds": 0.29514047899999696, "nonpadding_tokens_per_second": 213952.35317755464, "ranks": [{"compute_seconds": 0.2924605000000611, "nonpadding_tokens": 4676, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2882414289999815, "nonpadding_tokens": 5741, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2851050159999886, "nonpadding_tokens": 7122, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.291912041000046, "nonpadding_tokens": 9427, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2838235250000025, "nonpadding_tokens": 7319, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28747618900001726, "nonpadding_tokens": 9460, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28466854200007674, "nonpadding_tokens": 17636, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2922761789999413, "nonpadding_tokens": 1765, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003940540000257897, "prefetch": {"data_wait_seconds": 0.007259556000192902, "consumed_batches": 767, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1317, "tokens": 105144696, "loss_tokens": 63023, "loss": 2.52034401131333, "lr": 0.0002998053046341745, "optimizer_lrs": [0.019987020308944968, 0.0002998053046341745], "gradient_norm": 1.093791127204895, "seconds": 0.3668394219999982, "nonpadding_tokens_per_second": 171799.96538103887, "ranks": [{"compute_seconds": 0.36401045100001284, "nonpadding_tokens": 3140, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3599157249999507, "nonpadding_tokens": 3677, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3627493729999287, "nonpadding_tokens": 12408, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.35888993200001096, "nonpadding_tokens": 6746, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3632157099999631, "nonpadding_tokens": 4004, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3596928270000035, "nonpadding_tokens": 12963, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.35729931800005943, "nonpadding_tokens": 3365, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35787186799996107, "nonpadding_tokens": 16720, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004061519999822849, "prefetch": {"data_wait_seconds": 0.007270078000104263, "consumed_batches": 768, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1318, "tokens": 105231265, "loss_tokens": 86569, "loss": 2.689332353382851, "lr": 0.000299804908623866, "optimizer_lrs": [0.019986993908257734, 0.000299804908623866], "gradient_norm": 0.4968526065349579, "seconds": 0.32951169300008587, "nonpadding_tokens_per_second": 262719.0531899499, "ranks": [{"compute_seconds": 0.32023038900001666, "nonpadding_tokens": 8795, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3229611780000141, "nonpadding_tokens": 18827, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3262113559999307, "nonpadding_tokens": 4769, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32681927200007976, "nonpadding_tokens": 17499, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3186908320000157, "nonpadding_tokens": 7746, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3216461929999923, "nonpadding_tokens": 11626, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31892416599998796, "nonpadding_tokens": 12957, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31917217099999107, "nonpadding_tokens": 4350, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004730399999743895, "prefetch": {"data_wait_seconds": 0.007281613000102993, "consumed_batches": 769, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1319, "tokens": 105287380, "loss_tokens": 56115, "loss": 2.488209480531052, "lr": 0.0002998046517107588, "optimizer_lrs": [0.019986976780717255, 0.0002998046517107588], "gradient_norm": 0.47285303473472595, "seconds": 0.3185077350000256, "nonpadding_tokens_per_second": 176180.9646475163, "ranks": [{"compute_seconds": 0.3123224029999392, "nonpadding_tokens": 6911, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3117554389999668, "nonpadding_tokens": 2831, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3093033679999735, "nonpadding_tokens": 5253, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30955029300002934, "nonpadding_tokens": 11433, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31514478700000836, "nonpadding_tokens": 5117, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31278234299998076, "nonpadding_tokens": 8620, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3081917219999468, "nonpadding_tokens": 12590, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31564427299997533, "nonpadding_tokens": 3360, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0007045769999649565, "prefetch": {"data_wait_seconds": 0.007292185000167706, "consumed_batches": 770, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1320, "tokens": 105350878, "loss_tokens": 63498, "loss": 2.5841091943840753, "lr": 0.0002998043607920958, "optimizer_lrs": [0.01998695738613972, 0.0002998043607920958], "gradient_norm": 1.075074553489685, "seconds": 0.2662511270000323, "nonpadding_tokens_per_second": 238489.13135301883, "ranks": [{"compute_seconds": 0.26053976699995474, "nonpadding_tokens": 7511, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26149662200009516, "nonpadding_tokens": 2803, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.263817351000057, "nonpadding_tokens": 3637, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2594273820000126, "nonpadding_tokens": 22227, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25823287900004743, "nonpadding_tokens": 3306, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2624102979999634, "nonpadding_tokens": 9985, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2579455280000502, "nonpadding_tokens": 11692, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25865677299998424, "nonpadding_tokens": 2337, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040394400002696784, "prefetch": {"data_wait_seconds": 0.007303622000179644, "consumed_batches": 771, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1321, "tokens": 105434510, "loss_tokens": 83632, "loss": 2.6096759837741534, "lr": 0.0002998039772986382, "optimizer_lrs": [0.019986931819909213, 0.0002998039772986382], "gradient_norm": 0.5134890079498291, "seconds": 0.2989205989999846, "nonpadding_tokens_per_second": 279779.982643499, "ranks": [{"compute_seconds": 0.2963927700000113, "nonpadding_tokens": 2313, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2947735609999427, "nonpadding_tokens": 15545, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29485702600004515, "nonpadding_tokens": 5463, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29641811999999845, "nonpadding_tokens": 8192, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2909717369999498, "nonpadding_tokens": 8816, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29417211300005874, "nonpadding_tokens": 9830, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2956370460000244, "nonpadding_tokens": 23513, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2915578670000514, "nonpadding_tokens": 9960, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004131239999196623, "prefetch": {"data_wait_seconds": 0.007314141000236418, "consumed_batches": 772, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1322, "tokens": 105511036, "loss_tokens": 76526, "loss": 2.577955939811306, "lr": 0.0002998036260608977, "optimizer_lrs": [0.01998690840405985, 0.0002998036260608977], "gradient_norm": 0.47589632868766785, "seconds": 0.23591452699997717, "nonpadding_tokens_per_second": 324380.19384879764, "ranks": [{"compute_seconds": 0.23300576299993736, "nonpadding_tokens": 7644, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23007242300002417, "nonpadding_tokens": 1578, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23287009700004546, "nonpadding_tokens": 6090, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2334042859999954, "nonpadding_tokens": 19869, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23021309000000656, "nonpadding_tokens": 2603, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23333043100001305, "nonpadding_tokens": 18307, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23015683199992054, "nonpadding_tokens": 13099, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23049795800000084, "nonpadding_tokens": 7336, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004412509999838221, "prefetch": {"data_wait_seconds": 0.007323565000277199, "consumed_batches": 773, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1323, "tokens": 105586776, "loss_tokens": 75740, "loss": 2.6359108875759176, "lr": 0.00029980327812138325, "optimizer_lrs": [0.01998688520809222, 0.00029980327812138325], "gradient_norm": 0.6363702416419983, "seconds": 0.3272674029999507, "nonpadding_tokens_per_second": 231431.54284758205, "ranks": [{"compute_seconds": 0.32367121799995857, "nonpadding_tokens": 4507, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32118186100001367, "nonpadding_tokens": 3012, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3220241020000003, "nonpadding_tokens": 4907, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.324812325000039, "nonpadding_tokens": 11339, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3249828379999826, "nonpadding_tokens": 10981, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32408180600009473, "nonpadding_tokens": 14459, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3247745669999631, "nonpadding_tokens": 8680, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32207320700001674, "nonpadding_tokens": 17855, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003697920000149679, "prefetch": {"data_wait_seconds": 0.007332546000341722, "consumed_batches": 774, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1324, "tokens": 105642859, "loss_tokens": 56083, "loss": 2.750586184761871, "lr": 0.0002998030202852145, "optimizer_lrs": [0.019986868019014302, 0.0002998030202852145], "gradient_norm": 0.41107630729675293, "seconds": 0.2979575609999756, "nonpadding_tokens_per_second": 188224.79218778608, "ranks": [{"compute_seconds": 0.27293654300001435, "nonpadding_tokens": 9714, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2810978669999713, "nonpadding_tokens": 17440, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28176272600001084, "nonpadding_tokens": 3116, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28259451400003854, "nonpadding_tokens": 4242, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2812596429999985, "nonpadding_tokens": 4713, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2835519249999834, "nonpadding_tokens": 5181, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2844177539999464, "nonpadding_tokens": 8843, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28530873199997586, "nonpadding_tokens": 2834, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034170499998253945, "prefetch": {"data_wait_seconds": 0.00734131000042737, "consumed_batches": 775, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1325, "tokens": 105730187, "loss_tokens": 87328, "loss": 2.5555468678430744, "lr": 0.0002998026184672517, "optimizer_lrs": [0.019986841231150115, 0.0002998026184672517], "gradient_norm": 0.4450536370277405, "seconds": 0.4071549090000417, "nonpadding_tokens_per_second": 214483.47562473096, "ranks": [{"compute_seconds": 0.4018152490000375, "nonpadding_tokens": 11077, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4045878290000928, "nonpadding_tokens": 21346, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.40394464500002414, "nonpadding_tokens": 9571, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4042882840000175, "nonpadding_tokens": 10143, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.40481143199997405, "nonpadding_tokens": 14222, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4021163610000258, "nonpadding_tokens": 12327, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.4045957969999563, "nonpadding_tokens": 4336, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.40104810399998314, "nonpadding_tokens": 4306, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004020399999262736, "prefetch": {"data_wait_seconds": 0.007350026000452203, "consumed_batches": 776, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1326, "tokens": 105759031, "loss_tokens": 28844, "loss": 2.779858354423797, "lr": 0.00029980248565891505, "optimizer_lrs": [0.019986832377261003, 0.00029980248565891505], "gradient_norm": 0.4534134566783905, "seconds": 0.2430835699999534, "nonpadding_tokens_per_second": 118658.780599633, "ranks": [{"compute_seconds": 0.2409200060000103, "nonpadding_tokens": 4053, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23639205599999968, "nonpadding_tokens": 2549, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2373171369999909, "nonpadding_tokens": 4380, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2396354300000212, "nonpadding_tokens": 1968, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23612467500004186, "nonpadding_tokens": 5431, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23881636500004788, "nonpadding_tokens": 4646, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2399566249999907, "nonpadding_tokens": 3025, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24056133800002044, "nonpadding_tokens": 2792, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003631380000115314, "prefetch": {"data_wait_seconds": 0.007358343000490208, "consumed_batches": 777, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1327, "tokens": 105828509, "loss_tokens": 69478, "loss": 2.5691273226776823, "lr": 0.00029980216557353074, "optimizer_lrs": [0.019986811038235384, 0.00029980216557353074], "gradient_norm": 0.4149739146232605, "seconds": 0.27936692299999777, "nonpadding_tokens_per_second": 248698.0178394296, "ranks": [{"compute_seconds": 0.276526194999974, "nonpadding_tokens": 2012, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2764787469999419, "nonpadding_tokens": 12735, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2738591989999577, "nonpadding_tokens": 11058, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27611408899997514, "nonpadding_tokens": 2571, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2770743119999679, "nonpadding_tokens": 8568, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2739082680000138, "nonpadding_tokens": 11152, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27252121800006535, "nonpadding_tokens": 10165, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27325919099996554, "nonpadding_tokens": 11217, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000334872999928848, "prefetch": {"data_wait_seconds": 0.0073655810004567, "consumed_batches": 778, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1328, "tokens": 105885354, "loss_tokens": 56845, "loss": 2.554879771747735, "lr": 0.000299801903495809, "optimizer_lrs": [0.01998679356638727, 0.000299801903495809], "gradient_norm": 0.4060187339782715, "seconds": 0.5356184659999599, "nonpadding_tokens_per_second": 106129.6493836795, "ranks": [{"compute_seconds": 0.5316726319999816, "nonpadding_tokens": 10728, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5257242840000345, "nonpadding_tokens": 6945, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5326809960000674, "nonpadding_tokens": 2730, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.5332175700000334, "nonpadding_tokens": 13505, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.5300236459999041, "nonpadding_tokens": 1523, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5262039749999303, "nonpadding_tokens": 1645, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.5293384630000446, "nonpadding_tokens": 15538, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.5322398150000254, "nonpadding_tokens": 4231, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003763969999681649, "prefetch": {"data_wait_seconds": 0.007375649000508133, "consumed_batches": 779, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1329, "tokens": 105978418, "loss_tokens": 93064, "loss": 2.969862137883607, "lr": 0.000299801474060096, "optimizer_lrs": [0.019986764937339734, 0.000299801474060096], "gradient_norm": 0.3866782784461975, "seconds": 0.2564939210000148, "nonpadding_tokens_per_second": 362831.21111472516, "ranks": [{"compute_seconds": 0.25338321300000644, "nonpadding_tokens": 6831, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2498915759999818, "nonpadding_tokens": 8633, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2504785600000332, "nonpadding_tokens": 5434, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2512374099999306, "nonpadding_tokens": 7327, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2532272869999588, "nonpadding_tokens": 19194, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25225693599998067, "nonpadding_tokens": 18957, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2532045149999931, "nonpadding_tokens": 17051, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2541300629999341, "nonpadding_tokens": 9637, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040454100007991656, "prefetch": {"data_wait_seconds": 0.007384939000488089, "consumed_batches": 780, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1330, "tokens": 106056222, "loss_tokens": 77804, "loss": 2.890119523096499, "lr": 0.000299801114683733, "optimizer_lrs": [0.019986740978915535, 0.000299801114683733], "gradient_norm": 0.4315962493419647, "seconds": 0.3450915889999351, "nonpadding_tokens_per_second": 225458.98677355083, "ranks": [{"compute_seconds": 0.34208245099989654, "nonpadding_tokens": 19764, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3385239570000067, "nonpadding_tokens": 4201, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34146563000001606, "nonpadding_tokens": 3726, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34199552199993377, "nonpadding_tokens": 9530, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3421487350000234, "nonpadding_tokens": 12395, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3407417099999748, "nonpadding_tokens": 11696, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3420886359999713, "nonpadding_tokens": 12689, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3427010350000046, "nonpadding_tokens": 3803, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037628299992320535, "prefetch": {"data_wait_seconds": 0.007395204000502531, "consumed_batches": 781, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1331, "tokens": 106117544, "loss_tokens": 61322, "loss": 2.551364110759597, "lr": 0.00029980083120882304, "optimizer_lrs": [0.019986722080588205, 0.00029980083120882304], "gradient_norm": 0.3767876625061035, "seconds": 0.2022424699999874, "nonpadding_tokens_per_second": 303210.29999289376, "ranks": [{"compute_seconds": 0.165971820999971, "nonpadding_tokens": 9594, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.17496709499994267, "nonpadding_tokens": 5539, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.17353272700006528, "nonpadding_tokens": 3015, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.17409194300000763, "nonpadding_tokens": 11661, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.1746933590000026, "nonpadding_tokens": 6435, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.17371986500006642, "nonpadding_tokens": 6697, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.17446921900000234, "nonpadding_tokens": 15149, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.17133357199998045, "nonpadding_tokens": 3232, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.021761348000040925, "prefetch": {"data_wait_seconds": 0.007404539000461341, "consumed_batches": 782, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1332, "tokens": 106217536, "loss_tokens": 99992, "loss": 2.4788939365149214, "lr": 0.0002998003685403777, "optimizer_lrs": [0.019986691236025183, 0.0002998003685403777], "gradient_norm": 0.7214934229850769, "seconds": 0.29647367100005795, "nonpadding_tokens_per_second": 337271.09615740704, "ranks": [{"compute_seconds": 0.294132692000062, "nonpadding_tokens": 14990, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2934475819999989, "nonpadding_tokens": 13332, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29255986500004383, "nonpadding_tokens": 9706, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2929532749999453, "nonpadding_tokens": 6665, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2898754459999964, "nonpadding_tokens": 9934, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29277883500003554, "nonpadding_tokens": 12885, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28966512499994224, "nonpadding_tokens": 22406, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2941536119999455, "nonpadding_tokens": 10074, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034933100005218876, "prefetch": {"data_wait_seconds": 0.007412830000546933, "consumed_batches": 783, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1333, "tokens": 106264472, "loss_tokens": 46936, "loss": 2.635764491807994, "lr": 0.00029980015117996717, "optimizer_lrs": [0.019986676745331146, 0.00029980015117996717], "gradient_norm": 0.6237679719924927, "seconds": 0.2896694109999771, "nonpadding_tokens_per_second": 162032.98732154947, "ranks": [{"compute_seconds": 0.28672659599999406, "nonpadding_tokens": 8067, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2861828019999848, "nonpadding_tokens": 4059, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.283377277999989, "nonpadding_tokens": 4072, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28403266899999835, "nonpadding_tokens": 8048, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2863304390000394, "nonpadding_tokens": 4618, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28387144499993155, "nonpadding_tokens": 2453, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2822221950000312, "nonpadding_tokens": 6895, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.286713117999966, "nonpadding_tokens": 8724, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0006251269999211218, "prefetch": {"data_wait_seconds": 0.0074196300005979765, "consumed_batches": 784, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1334, "tokens": 106338321, "loss_tokens": 73849, "loss": 2.6331750429931344, "lr": 0.0002997998089463176, "optimizer_lrs": [0.01998665392975451, 0.0002997998089463176], "gradient_norm": 0.40031370520591736, "seconds": 0.2800950120000607, "nonpadding_tokens_per_second": 263656.96223103034, "ranks": [{"compute_seconds": 0.27764042000001155, "nonpadding_tokens": 4181, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2778607050000801, "nonpadding_tokens": 9479, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2770098150000422, "nonpadding_tokens": 11305, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2753703989999394, "nonpadding_tokens": 1551, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2738746469999569, "nonpadding_tokens": 5835, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2754115789999787, "nonpadding_tokens": 20052, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2777427880000687, "nonpadding_tokens": 18814, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27458234400000947, "nonpadding_tokens": 2632, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003732339999942269, "prefetch": {"data_wait_seconds": 0.007428366000638675, "consumed_batches": 785, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1335, "tokens": 106386233, "loss_tokens": 47912, "loss": 2.5931323311487726, "lr": 0.00029979958675435355, "optimizer_lrs": [0.019986639116956906, 0.00029979958675435355], "gradient_norm": 0.4595867097377777, "seconds": 0.26000812700010556, "nonpadding_tokens_per_second": 184271.16318552822, "ranks": [{"compute_seconds": 0.2574603369999977, "nonpadding_tokens": 5189, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25601551800002653, "nonpadding_tokens": 3668, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2534770340000705, "nonpadding_tokens": 3850, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25711219700008314, "nonpadding_tokens": 4134, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25240853000002517, "nonpadding_tokens": 3521, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25549653500002023, "nonpadding_tokens": 7171, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.25692845300000045, "nonpadding_tokens": 8091, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25771155999996154, "nonpadding_tokens": 12288, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040707100004055974, "prefetch": {"data_wait_seconds": 0.007438354000626646, "consumed_batches": 786, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1336, "tokens": 106455081, "loss_tokens": 68848, "loss": 2.5006186635777365, "lr": 0.00029979926725604353, "optimizer_lrs": [0.01998661781706957, 0.00029979926725604353], "gradient_norm": 0.397373229265213, "seconds": 0.3402380060000496, "nonpadding_tokens_per_second": 202352.46734895912, "ranks": [{"compute_seconds": 0.3377583380000715, "nonpadding_tokens": 14817, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3345688519999612, "nonpadding_tokens": 8400, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33548445400003857, "nonpadding_tokens": 10563, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33792198999992706, "nonpadding_tokens": 5885, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3343238489999294, "nonpadding_tokens": 8106, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3360061900000346, "nonpadding_tokens": 9280, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.33417778499995165, "nonpadding_tokens": 3924, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3349528470000678, "nonpadding_tokens": 7873, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003865760000962837, "prefetch": {"data_wait_seconds": 0.007446817000598571, "consumed_batches": 787, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1337, "tokens": 106518616, "loss_tokens": 63535, "loss": 2.5166706834815455, "lr": 0.0002997989721878149, "optimizer_lrs": [0.019986598145854326, 0.0002997989721878149], "gradient_norm": 0.39777541160583496, "seconds": 0.32452271399995425, "nonpadding_tokens_per_second": 195779.8245210317, "ranks": [{"compute_seconds": 0.2945633539999335, "nonpadding_tokens": 9978, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29493237200006206, "nonpadding_tokens": 8603, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29418582899995727, "nonpadding_tokens": 5253, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2945802360000016, "nonpadding_tokens": 7936, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2952184580000221, "nonpadding_tokens": 5884, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.293603914000073, "nonpadding_tokens": 6670, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.295039609000014, "nonpadding_tokens": 9778, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29185762200006593, "nonpadding_tokens": 9433, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0030231040000217035, "prefetch": {"data_wait_seconds": 0.007455127000596917, "consumed_batches": 788, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1338, "tokens": 106588594, "loss_tokens": 69978, "loss": 2.7095310311812284, "lr": 0.00029979864694650797, "optimizer_lrs": [0.019986576463100533, 0.00029979864694650797], "gradient_norm": 0.3698655068874359, "seconds": 0.32339347100003124, "nonpadding_tokens_per_second": 216386.5577854948, "ranks": [{"compute_seconds": 0.32106794799994987, "nonpadding_tokens": 4387, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31990952799992556, "nonpadding_tokens": 3963, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3173789569999599, "nonpadding_tokens": 14535, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3197389680000242, "nonpadding_tokens": 17148, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32009940199998255, "nonpadding_tokens": 6654, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3187988369999175, "nonpadding_tokens": 16844, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.316106450999996, "nonpadding_tokens": 2771, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31662831899996036, "nonpadding_tokens": 3676, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000340905000030034, "prefetch": {"data_wait_seconds": 0.007463616000521256, "consumed_batches": 789, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1339, "tokens": 106649412, "loss_tokens": 60818, "loss": 2.5958942048406723, "lr": 0.00029979836406536444, "optimizer_lrs": [0.019986557604357632, 0.00029979836406536444], "gradient_norm": 0.8283106684684753, "seconds": 0.2535778690000825, "nonpadding_tokens_per_second": 239839.54214861002, "ranks": [{"compute_seconds": 0.2507899649999672, "nonpadding_tokens": 7810, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24905222699999285, "nonpadding_tokens": 7580, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24803207499996915, "nonpadding_tokens": 7909, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25056593800002247, "nonpadding_tokens": 4480, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25096414600000116, "nonpadding_tokens": 2166, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24961246700001993, "nonpadding_tokens": 15974, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.25097300100003395, "nonpadding_tokens": 5439, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2513220990000491, "nonpadding_tokens": 9460, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003614770000694989, "prefetch": {"data_wait_seconds": 0.007472705000466817, "consumed_batches": 790, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1340, "tokens": 106724582, "loss_tokens": 75170, "loss": 2.5457928694958096, "lr": 0.0002997980141549624, "optimizer_lrs": [0.019986534276997495, 0.0002997980141549624], "gradient_norm": 0.403221994638443, "seconds": 0.34617174000004525, "nonpadding_tokens_per_second": 217146.55274861597, "ranks": [{"compute_seconds": 0.3427735360000952, "nonpadding_tokens": 9123, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33976734500004113, "nonpadding_tokens": 15804, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34286096099992847, "nonpadding_tokens": 7633, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3412367879999465, "nonpadding_tokens": 5151, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3438656280000032, "nonpadding_tokens": 4224, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34072578200004955, "nonpadding_tokens": 7696, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.33987152899999273, "nonpadding_tokens": 9170, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34064286000000266, "nonpadding_tokens": 16369, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038980800002264004, "prefetch": {"data_wait_seconds": 0.0074823300004283055, "consumed_batches": 791, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1341, "tokens": 106823184, "loss_tokens": 98602, "loss": 2.1844772735340054, "lr": 0.00029979755471087573, "optimizer_lrs": [0.019986503647391716, 0.00029979755471087573], "gradient_norm": 0.6130238771438599, "seconds": 0.36600322099991445, "nonpadding_tokens_per_second": 269402.00069994206, "ranks": [{"compute_seconds": 0.3626317250000284, "nonpadding_tokens": 6879, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35941502799994396, "nonpadding_tokens": 7127, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36220856499994625, "nonpadding_tokens": 25143, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3613574759999665, "nonpadding_tokens": 19537, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.36338383700001486, "nonpadding_tokens": 9844, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36069170600001144, "nonpadding_tokens": 10649, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.36304366499996377, "nonpadding_tokens": 2549, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3636957490000441, "nonpadding_tokens": 16874, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033879099999012396, "prefetch": {"data_wait_seconds": 0.007490511000426636, "consumed_batches": 792, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1342, "tokens": 106921499, "loss_tokens": 98315, "loss": 2.4719808460051875, "lr": 0.0002997970960848156, "optimizer_lrs": [0.01998647307232104, 0.0002997970960848156], "gradient_norm": 0.5101664066314697, "seconds": 0.3283349390000012, "nonpadding_tokens_per_second": 299435.08387939073, "ranks": [{"compute_seconds": 0.32537798800001383, "nonpadding_tokens": 10692, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32594544200003384, "nonpadding_tokens": 11517, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32460603399999854, "nonpadding_tokens": 9724, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32356065900000885, "nonpadding_tokens": 5984, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32174329500003296, "nonpadding_tokens": 2180, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.323060541000018, "nonpadding_tokens": 16044, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3254680610000378, "nonpadding_tokens": 20414, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32609576100003324, "nonpadding_tokens": 21760, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003319830000236834, "prefetch": {"data_wait_seconds": 0.007498457000409076, "consumed_batches": 793, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1343, "tokens": 106974492, "loss_tokens": 52993, "loss": 2.8635919484648915, "lr": 0.00029979684866463516, "optimizer_lrs": [0.019986456577642347, 0.00029979684866463516], "gradient_norm": 0.4514543116092682, "seconds": 0.35132796099992447, "nonpadding_tokens_per_second": 150836.2723228038, "ranks": [{"compute_seconds": 0.3481814489999806, "nonpadding_tokens": 3480, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34492767400001867, "nonpadding_tokens": 2718, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3456059310000228, "nonpadding_tokens": 5519, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3480400940001118, "nonpadding_tokens": 1999, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34468457899993155, "nonpadding_tokens": 8167, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3473683209999763, "nonpadding_tokens": 13545, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.34821529099997406, "nonpadding_tokens": 11360, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.348943405, "nonpadding_tokens": 6205, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039308399993842613, "prefetch": {"data_wait_seconds": 0.00750655500037567, "consumed_batches": 794, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1344, "tokens": 107056020, "loss_tokens": 81528, "loss": 2.4920132883794524, "lr": 0.0002997964677226045, "optimizer_lrs": [0.019986431181506968, 0.0002997964677226045], "gradient_norm": 0.3615008294582367, "seconds": 0.266115128000024, "nonpadding_tokens_per_second": 306363.6427313243, "ranks": [{"compute_seconds": 0.2625063739999405, "nonpadding_tokens": 12646, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2630121219999637, "nonpadding_tokens": 24655, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2625340229999438, "nonpadding_tokens": 5502, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2626125070000853, "nonpadding_tokens": 2728, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25919826000006196, "nonpadding_tokens": 10181, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2616751040000054, "nonpadding_tokens": 5254, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.25913208600002235, "nonpadding_tokens": 4806, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2637034400000857, "nonpadding_tokens": 15756, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003572360000134722, "prefetch": {"data_wait_seconds": 0.007515170000445437, "consumed_batches": 795, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1345, "tokens": 107170963, "loss_tokens": 114943, "loss": 2.859691597139452, "lr": 0.00029979593004226396, "optimizer_lrs": [0.01998639533615093, 0.00029979593004226396], "gradient_norm": 0.40267428755760193, "seconds": 0.24359762199992474, "nonpadding_tokens_per_second": 471856.00194420415, "ranks": [{"compute_seconds": 0.24072543599993423, "nonpadding_tokens": 2992, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24129034999998566, "nonpadding_tokens": 18888, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2395019200000661, "nonpadding_tokens": 11205, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2385070340000084, "nonpadding_tokens": 21338, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24092233900000792, "nonpadding_tokens": 21633, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23823809099997106, "nonpadding_tokens": 18017, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23642265600005885, "nonpadding_tokens": 6431, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23753243900000598, "nonpadding_tokens": 14439, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037036999992778874, "prefetch": {"data_wait_seconds": 0.007524348000472969, "consumed_batches": 796, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1346, "tokens": 107251115, "loss_tokens": 80152, "loss": 2.7635979919403133, "lr": 0.00029979555468783635, "optimizer_lrs": [0.019986370312522424, 0.00029979555468783635], "gradient_norm": 0.4304940104484558, "seconds": 0.3791346519999479, "nonpadding_tokens_per_second": 211407.74017145502, "ranks": [{"compute_seconds": 0.37616145600009077, "nonpadding_tokens": 13887, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37283180899999024, "nonpadding_tokens": 4309, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37318817699997453, "nonpadding_tokens": 4961, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3759844850000036, "nonpadding_tokens": 9984, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3763297320000447, "nonpadding_tokens": 24773, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37526176200003647, "nonpadding_tokens": 15870, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3717917879999959, "nonpadding_tokens": 3167, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3767919269999993, "nonpadding_tokens": 3201, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037765299998682167, "prefetch": {"data_wait_seconds": 0.007532434000452781, "consumed_batches": 797, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1347, "tokens": 107323065, "loss_tokens": 71950, "loss": 2.5007985145934675, "lr": 0.000299795217450112, "optimizer_lrs": [0.019986347830007467, 0.000299795217450112], "gradient_norm": 0.41548311710357666, "seconds": 0.2686155430000099, "nonpadding_tokens_per_second": 267854.93942916533, "ranks": [{"compute_seconds": 0.26348945899997034, "nonpadding_tokens": 9544, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2662940359999766, "nonpadding_tokens": 9462, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2627601159999813, "nonpadding_tokens": 5089, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26360379499999453, "nonpadding_tokens": 10354, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2660452830000395, "nonpadding_tokens": 6035, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2630143540000063, "nonpadding_tokens": 9942, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2614874719999989, "nonpadding_tokens": 7754, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2622312949999923, "nonpadding_tokens": 13770, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038299899995308806, "prefetch": {"data_wait_seconds": 0.007540784000482859, "consumed_batches": 798, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1348, "tokens": 107366418, "loss_tokens": 43353, "loss": 2.7127509197748716, "lr": 0.00029979501411566155, "optimizer_lrs": [0.01998633427437744, 0.00029979501411566155], "gradient_norm": 0.4909362196922302, "seconds": 0.28189122800006317, "nonpadding_tokens_per_second": 153793.3631620147, "ranks": [{"compute_seconds": 0.22737344999995912, "nonpadding_tokens": 1880, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22723546200006695, "nonpadding_tokens": 7137, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22660181200001261, "nonpadding_tokens": 7119, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2303524669999888, "nonpadding_tokens": 3732, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23090077099993778, "nonpadding_tokens": 3408, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2289729440000201, "nonpadding_tokens": 4706, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.22533555600000454, "nonpadding_tokens": 10753, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2262364550000484, "nonpadding_tokens": 4618, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.033024707999970815, "prefetch": {"data_wait_seconds": 0.007549384000483883, "consumed_batches": 799, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1349, "tokens": 107421476, "loss_tokens": 55058, "loss": 2.752288779559737, "lr": 0.00029979475573703565, "optimizer_lrs": [0.019986317049135713, 0.00029979475573703565], "gradient_norm": 0.45732152462005615, "seconds": 0.28411784600007195, "nonpadding_tokens_per_second": 193785.78563483147, "ranks": [{"compute_seconds": 0.280182018000005, "nonpadding_tokens": 12281, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2779637300000104, "nonpadding_tokens": 11474, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2784481209999967, "nonpadding_tokens": 7449, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27839859100004105, "nonpadding_tokens": 2731, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.281713730999968, "nonpadding_tokens": 1701, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2800159659999508, "nonpadding_tokens": 10412, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27681134899989956, "nonpadding_tokens": 7240, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2773081819999561, "nonpadding_tokens": 1770, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038381699994261, "prefetch": {"data_wait_seconds": 0.007558468000482321, "consumed_batches": 800, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1350, "tokens": 107523098, "loss_tokens": 101622, "loss": 2.331038677648541, "lr": 0.00029979427841374766, "optimizer_lrs": [0.019986285227583177, 0.00029979427841374766], "gradient_norm": 0.48051533102989197, "seconds": 0.35639259899994613, "nonpadding_tokens_per_second": 285140.601362531, "ranks": [{"compute_seconds": 0.34867042299993045, "nonpadding_tokens": 16032, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3508227080000097, "nonpadding_tokens": 3524, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34811335300003066, "nonpadding_tokens": 14233, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.35373303900007613, "nonpadding_tokens": 9201, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.34707343599995966, "nonpadding_tokens": 2160, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3528877049999437, "nonpadding_tokens": 20659, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3540420699999913, "nonpadding_tokens": 16800, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3479965120000088, "nonpadding_tokens": 19013, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034858700007589505, "prefetch": {"data_wait_seconds": 0.007567085000417251, "consumed_batches": 801, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1351, "tokens": 107584100, "loss_tokens": 61002, "loss": 2.8242785584898855, "lr": 0.0002997939916184204, "optimizer_lrs": [0.019986266107894696, 0.0002997939916184204], "gradient_norm": 0.4463610053062439, "seconds": 0.3750365810000176, "nonpadding_tokens_per_second": 162656.13300265538, "ranks": [{"compute_seconds": 0.37279745599994385, "nonpadding_tokens": 4350, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3669812589999992, "nonpadding_tokens": 5588, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3694685870000285, "nonpadding_tokens": 14185, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3698255680000102, "nonpadding_tokens": 8440, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3703004349999901, "nonpadding_tokens": 4767, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3678872010000305, "nonpadding_tokens": 6831, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.36986551899997266, "nonpadding_tokens": 10845, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3689707780000617, "nonpadding_tokens": 5996, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037743000007139926, "prefetch": {"data_wait_seconds": 0.007575173000489599, "consumed_batches": 802, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1352, "tokens": 107647658, "loss_tokens": 63558, "loss": 2.9014374665659712, "lr": 0.00029979369259395346, "optimizer_lrs": [0.019986246172930232, 0.00029979369259395346], "gradient_norm": 0.4032559394836426, "seconds": 0.2827311999999438, "nonpadding_tokens_per_second": 224800.0928090449, "ranks": [{"compute_seconds": 0.2777932479999663, "nonpadding_tokens": 13943, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2767976359999693, "nonpadding_tokens": 10022, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2791852010000184, "nonpadding_tokens": 9685, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2774373770000693, "nonpadding_tokens": 1238, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2755822780000017, "nonpadding_tokens": 8601, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2787322229999063, "nonpadding_tokens": 4596, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28030354400004853, "nonpadding_tokens": 4772, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27667433399994934, "nonpadding_tokens": 10701, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042751799992402084, "prefetch": {"data_wait_seconds": 0.007585101000429972, "consumed_batches": 803, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1353, "tokens": 107715002, "loss_tokens": 67344, "loss": 2.4748578659568783, "lr": 0.0002997933755208624, "optimizer_lrs": [0.019986225034724164, 0.0002997933755208624], "gradient_norm": 0.3775898218154907, "seconds": 0.3231020919999992, "nonpadding_tokens_per_second": 208429.47683545228, "ranks": [{"compute_seconds": 0.31865652900000896, "nonpadding_tokens": 14431, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3207493230000864, "nonpadding_tokens": 8389, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31752586200002497, "nonpadding_tokens": 6384, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31861940299995695, "nonpadding_tokens": 2797, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31910902999993596, "nonpadding_tokens": 9169, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31617913299999145, "nonpadding_tokens": 10805, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31918210000003455, "nonpadding_tokens": 11282, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3169591919998993, "nonpadding_tokens": 4087, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034922100007861445, "prefetch": {"data_wait_seconds": 0.007593912000473324, "consumed_batches": 804, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1354, "tokens": 107790298, "loss_tokens": 75296, "loss": 2.7161153978963024, "lr": 0.0002997930207196178, "optimizer_lrs": [0.019986201381307852, 0.0002997930207196178], "gradient_norm": 0.3729287385940552, "seconds": 0.28274501199996394, "nonpadding_tokens_per_second": 266303.548442473, "ranks": [{"compute_seconds": 0.2782441800000015, "nonpadding_tokens": 6634, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27628156000002946, "nonpadding_tokens": 1319, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27690880000000107, "nonpadding_tokens": 2662, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2795797630000152, "nonpadding_tokens": 8704, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2802052100000765, "nonpadding_tokens": 13156, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2773062769999797, "nonpadding_tokens": 21613, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27614395399996283, "nonpadding_tokens": 2748, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28043273900004806, "nonpadding_tokens": 18460, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039416800007074926, "prefetch": {"data_wait_seconds": 0.007604102000485113, "consumed_batches": 805, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1355, "tokens": 107850530, "loss_tokens": 60232, "loss": 2.6834030810034535, "lr": 0.0002997927366822952, "optimizer_lrs": [0.019986182445486347, 0.0002997927366822952], "gradient_norm": 0.412250816822052, "seconds": 0.35286247099998036, "nonpadding_tokens_per_second": 170695.39820799857, "ranks": [{"compute_seconds": 0.34997313500002747, "nonpadding_tokens": 10680, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34710328900007426, "nonpadding_tokens": 7829, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3450504719999117, "nonpadding_tokens": 8561, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3445247479999125, "nonpadding_tokens": 3161, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3503267830000141, "nonpadding_tokens": 17423, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34597448499994243, "nonpadding_tokens": 6359, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.34998661899999206, "nonpadding_tokens": 4784, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35022629000002325, "nonpadding_tokens": 1435, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039121800000430085, "prefetch": {"data_wait_seconds": 0.00761282600053903, "consumed_batches": 806, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1356, "tokens": 107906112, "loss_tokens": 55582, "loss": 2.618921986884243, "lr": 0.0002997924744004239, "optimizer_lrs": [0.019986164960028262, 0.0002997924744004239], "gradient_norm": 0.44682222604751587, "seconds": 0.30872260799992546, "nonpadding_tokens_per_second": 180038.64491846162, "ranks": [{"compute_seconds": 0.3052804729999252, "nonpadding_tokens": 3145, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3014387270000043, "nonpadding_tokens": 6250, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3061357980000139, "nonpadding_tokens": 3562, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30014312100001916, "nonpadding_tokens": 17924, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2982478420000234, "nonpadding_tokens": 6212, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3025075379999862, "nonpadding_tokens": 10657, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2984798210000008, "nonpadding_tokens": 6345, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3060837589999892, "nonpadding_tokens": 1487, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035656100010328373, "prefetch": {"data_wait_seconds": 0.007620811000563776, "consumed_batches": 807, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1357, "tokens": 107963309, "loss_tokens": 57197, "loss": 2.783368500533245, "lr": 0.00029979220432463344, "optimizer_lrs": [0.019986146954975564, 0.00029979220432463344], "gradient_norm": 0.4021473228931427, "seconds": 0.32870598300007714, "nonpadding_tokens_per_second": 174006.56805199245, "ranks": [{"compute_seconds": 0.3262276059999749, "nonpadding_tokens": 5912, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32352791000005254, "nonpadding_tokens": 9477, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3234685830000217, "nonpadding_tokens": 12380, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32628993899993475, "nonpadding_tokens": 7974, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32236027600004036, "nonpadding_tokens": 4664, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32398457400006464, "nonpadding_tokens": 4367, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.32207227399999283, "nonpadding_tokens": 3620, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32307818499998575, "nonpadding_tokens": 8803, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039050400005180563, "prefetch": {"data_wait_seconds": 0.0076315980006285145, "consumed_batches": 808, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1358, "tokens": 108024141, "loss_tokens": 60832, "loss": 2.5390794524263547, "lr": 0.00029979191689233597, "optimizer_lrs": [0.0199861277928224, 0.00029979191689233597], "gradient_norm": 0.4054037034511566, "seconds": 0.39103687900001205, "nonpadding_tokens_per_second": 155565.88973286617, "ranks": [{"compute_seconds": 0.3862052759999415, "nonpadding_tokens": 15849, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3842207299999245, "nonpadding_tokens": 4417, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38333599500003857, "nonpadding_tokens": 6487, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3885312799999383, "nonpadding_tokens": 2161, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.38861427800009096, "nonpadding_tokens": 5184, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3853425900000502, "nonpadding_tokens": 3066, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3827701910000769, "nonpadding_tokens": 13359, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.38353395000001456, "nonpadding_tokens": 10309, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036079599999538914, "prefetch": {"data_wait_seconds": 0.0076407430005502874, "consumed_batches": 809, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1359, "tokens": 108099767, "loss_tokens": 75626, "loss": 2.4530097122682677, "lr": 0.00029979155928133516, "optimizer_lrs": [0.019986103952089013, 0.00029979155928133516], "gradient_norm": 0.3652321994304657, "seconds": 0.3002028280000104, "nonpadding_tokens_per_second": 251916.34770341797, "ranks": [{"compute_seconds": 0.2969457970000349, "nonpadding_tokens": 9048, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29773787499993887, "nonpadding_tokens": 11002, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2967121709999674, "nonpadding_tokens": 4161, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2950956249999308, "nonpadding_tokens": 12143, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29727570399995784, "nonpadding_tokens": 4857, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2945822899999939, "nonpadding_tokens": 3617, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2934671120000303, "nonpadding_tokens": 14686, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29789803799997117, "nonpadding_tokens": 16112, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035498200008987624, "prefetch": {"data_wait_seconds": 0.007650120000519678, "consumed_batches": 810, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1360, "tokens": 108179121, "loss_tokens": 79354, "loss": 2.730844735300048, "lr": 0.00029979118371197146, "optimizer_lrs": [0.01998607891413143, 0.00029979118371197146], "gradient_norm": 0.3673522174358368, "seconds": 0.24652722999996968, "nonpadding_tokens_per_second": 321887.3631120171, "ranks": [{"compute_seconds": 0.24354575500001374, "nonpadding_tokens": 9837, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24282600200001525, "nonpadding_tokens": 5865, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.242655672000069, "nonpadding_tokens": 3977, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23995602800005145, "nonpadding_tokens": 6809, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2441007749999926, "nonpadding_tokens": 19014, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.241195394999977, "nonpadding_tokens": 2856, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24341282199998204, "nonpadding_tokens": 11485, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24365083300006063, "nonpadding_tokens": 19511, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003841340000008131, "prefetch": {"data_wait_seconds": 0.007658990000507515, "consumed_batches": 811, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1361, "tokens": 108281221, "loss_tokens": 102100, "loss": 2.824645568070519, "lr": 0.0002997906999926549, "optimizer_lrs": [0.019986046666176996, 0.0002997906999926549], "gradient_norm": 0.3290064334869385, "seconds": 0.328895702000068, "nonpadding_tokens_per_second": 310432.7584067331, "ranks": [{"compute_seconds": 0.2741205859999809, "nonpadding_tokens": 26867, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29232103800006826, "nonpadding_tokens": 12915, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2947922479999079, "nonpadding_tokens": 7935, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29532140300000265, "nonpadding_tokens": 8247, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2960836610000115, "nonpadding_tokens": 8054, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29215154500002427, "nonpadding_tokens": 9569, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.29545651399996586, "nonpadding_tokens": 13066, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2963956410000037, "nonpadding_tokens": 15447, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.019852945000025102, "prefetch": {"data_wait_seconds": 0.007668830000582716, "consumed_batches": 812, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1362, "tokens": 108387221, "loss_tokens": 106000, "loss": 2.309949882075472, "lr": 0.0002997901972046745, "optimizer_lrs": [0.0199860131469783, 0.0002997901972046745], "gradient_norm": 0.6513566970825195, "seconds": 0.3240523359999088, "nonpadding_tokens_per_second": 327107.65584491834, "ranks": [{"compute_seconds": 0.3203568029999815, "nonpadding_tokens": 12472, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31813334100002066, "nonpadding_tokens": 12266, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32086101999993843, "nonpadding_tokens": 17691, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31726870500006044, "nonpadding_tokens": 25830, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32167203500000596, "nonpadding_tokens": 10378, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31926789599992844, "nonpadding_tokens": 4359, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31689472000005026, "nonpadding_tokens": 14992, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3160761020000109, "nonpadding_tokens": 8012, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000345487000004141, "prefetch": {"data_wait_seconds": 0.007677956000591735, "consumed_batches": 813, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1363, "tokens": 108436459, "loss_tokens": 49238, "loss": 2.7366782261667817, "lr": 0.0002997899634499076, "optimizer_lrs": [0.019985997563327175, 0.0002997899634499076], "gradient_norm": 0.4460412859916687, "seconds": 0.2557902290000129, "nonpadding_tokens_per_second": 192493.67027228203, "ranks": [{"compute_seconds": 0.2525283910000553, "nonpadding_tokens": 3140, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24951041099996019, "nonpadding_tokens": 5080, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2491709800000308, "nonpadding_tokens": 2356, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2530779989999701, "nonpadding_tokens": 8391, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25337375399999473, "nonpadding_tokens": 5917, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25092783699994925, "nonpadding_tokens": 10844, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24794488199995612, "nonpadding_tokens": 9831, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24869452699999783, "nonpadding_tokens": 3679, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037995700006376865, "prefetch": {"data_wait_seconds": 0.007686215000560992, "consumed_batches": 814, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1364, "tokens": 108475321, "loss_tokens": 38862, "loss": 2.8652680478359325, "lr": 0.0002997897788628179, "optimizer_lrs": [0.019985985257521193, 0.0002997897788628179], "gradient_norm": 0.578760027885437, "seconds": 0.26562536599999476, "nonpadding_tokens_per_second": 146303.7984105809, "ranks": [{"compute_seconds": 0.2609600270000101, "nonpadding_tokens": 7103, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2610402039999826, "nonpadding_tokens": 4408, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26176581599997917, "nonpadding_tokens": 4579, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2623600860000579, "nonpadding_tokens": 5959, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2628707520000262, "nonpadding_tokens": 3655, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2592231419999962, "nonpadding_tokens": 2366, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26288466300002256, "nonpadding_tokens": 5364, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26323331699995833, "nonpadding_tokens": 5428, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037317200008146756, "prefetch": {"data_wait_seconds": 0.007695242000593794, "consumed_batches": 815, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1365, "tokens": 108515794, "loss_tokens": 40473, "loss": 2.8672569907716254, "lr": 0.0002997895865376665, "optimizer_lrs": [0.019985972435844435, 0.0002997895865376665], "gradient_norm": 0.43678516149520874, "seconds": 0.2828492469999446, "nonpadding_tokens_per_second": 143090.35795314642, "ranks": [{"compute_seconds": 0.2758436540000275, "nonpadding_tokens": 2801, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27590246199997637, "nonpadding_tokens": 6350, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2750305179999941, "nonpadding_tokens": 3580, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2760057189999543, "nonpadding_tokens": 2052, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.274029282000015, "nonpadding_tokens": 12206, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27719347200002176, "nonpadding_tokens": 3991, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.274481053000045, "nonpadding_tokens": 6825, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2803208770000083, "nonpadding_tokens": 2668, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044341199998143566, "prefetch": {"data_wait_seconds": 0.007704696000544686, "consumed_batches": 816, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1366, "tokens": 108563710, "loss_tokens": 47916, "loss": 2.83140026296018, "lr": 0.00029978935873025553, "optimizer_lrs": [0.019985957248683704, 0.00029978935873025553], "gradient_norm": 0.4012396037578583, "seconds": 0.2554225690000749, "nonpadding_tokens_per_second": 187595.01240466326, "ranks": [{"compute_seconds": 0.2525615550000566, "nonpadding_tokens": 7223, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24903333600002497, "nonpadding_tokens": 10562, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24606701300001532, "nonpadding_tokens": 4755, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24706856500006324, "nonpadding_tokens": 9821, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24477591799995935, "nonpadding_tokens": 2351, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24815515399996002, "nonpadding_tokens": 3647, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.252035658000068, "nonpadding_tokens": 7586, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25287401100001716, "nonpadding_tokens": 1971, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039594399993347906, "prefetch": {"data_wait_seconds": 0.007713726000545762, "consumed_batches": 817, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1367, "tokens": 108627414, "loss_tokens": 63704, "loss": 3.0031696891090043, "lr": 0.00029978905567112267, "optimizer_lrs": [0.01998593704474151, 0.00029978905567112267], "gradient_norm": 0.41105708479881287, "seconds": 0.2811975209999673, "nonpadding_tokens_per_second": 226545.38266718006, "ranks": [{"compute_seconds": 0.2776564439999447, "nonpadding_tokens": 6417, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.274548702000061, "nonpadding_tokens": 4876, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27692985999999564, "nonpadding_tokens": 3974, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27806289000000106, "nonpadding_tokens": 16725, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27248589399994216, "nonpadding_tokens": 10874, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27565356299999166, "nonpadding_tokens": 7535, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27211845800002266, "nonpadding_tokens": 11519, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2787153510000735, "nonpadding_tokens": 1784, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040616700005102757, "prefetch": {"data_wait_seconds": 0.007723476000549567, "consumed_batches": 818, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1368, "tokens": 108697115, "loss_tokens": 69701, "loss": 2.688577818108779, "lr": 0.00029978872383306423, "optimizer_lrs": [0.019985914922204282, 0.00029978872383306423], "gradient_norm": 0.4030831456184387, "seconds": 0.361225887000046, "nonpadding_tokens_per_second": 192956.824270989, "ranks": [{"compute_seconds": 0.3543271129999539, "nonpadding_tokens": 7886, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35517144200002804, "nonpadding_tokens": 4505, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3520009740000205, "nonpadding_tokens": 17850, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3579258319999781, "nonpadding_tokens": 10067, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.35153014299999086, "nonpadding_tokens": 8331, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3557017850000648, "nonpadding_tokens": 3847, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.35792358899993815, "nonpadding_tokens": 11872, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3586921270000403, "nonpadding_tokens": 5343, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046155200004704966, "prefetch": {"data_wait_seconds": 0.007733945000566678, "consumed_batches": 819, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1369, "tokens": 108770680, "loss_tokens": 73565, "loss": 2.839342970502277, "lr": 0.00029978837331629806, "optimizer_lrs": [0.019985891554419873, 0.00029978837331629806], "gradient_norm": 0.9373819231987, "seconds": 0.33982453099997656, "nonpadding_tokens_per_second": 216479.36887759605, "ranks": [{"compute_seconds": 0.3357329259999915, "nonpadding_tokens": 26731, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33238213200002065, "nonpadding_tokens": 11018, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3327672119999079, "nonpadding_tokens": 16247, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3328797609999583, "nonpadding_tokens": 2651, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33631625400005305, "nonpadding_tokens": 2959, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3334323269999686, "nonpadding_tokens": 4226, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.33619346900002256, "nonpadding_tokens": 2989, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33716895700001714, "nonpadding_tokens": 6744, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035168700003396225, "prefetch": {"data_wait_seconds": 0.007742503000557122, "consumed_batches": 820, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1370, "tokens": 108824462, "loss_tokens": 53782, "loss": 2.730017419861664, "lr": 0.0002997881168763336, "optimizer_lrs": [0.019985874458422242, 0.0002997881168763336], "gradient_norm": 0.5266878008842468, "seconds": 0.2807358820000445, "nonpadding_tokens_per_second": 191575.08337317378, "ranks": [{"compute_seconds": 0.2777253589999873, "nonpadding_tokens": 2396, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27437591599993993, "nonpadding_tokens": 19657, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27384592199996405, "nonpadding_tokens": 3051, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2779212240000106, "nonpadding_tokens": 2507, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27237434800008486, "nonpadding_tokens": 5746, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27510653599995294, "nonpadding_tokens": 13760, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2725303199999871, "nonpadding_tokens": 2028, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27819069700001364, "nonpadding_tokens": 4637, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000420539999936409, "prefetch": {"data_wait_seconds": 0.007752359000576803, "consumed_batches": 821, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1371, "tokens": 108879257, "loss_tokens": 54795, "loss": 2.4146038644036865, "lr": 0.0002997878554466795, "optimizer_lrs": [0.019985857029778637, 0.0002997878554466795], "gradient_norm": 0.5610395073890686, "seconds": 0.2769797070000095, "nonpadding_tokens_per_second": 197830.37751570053, "ranks": [{"compute_seconds": 0.27458511099996485, "nonpadding_tokens": 8828, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2723801900000353, "nonpadding_tokens": 10076, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27269591300000684, "nonpadding_tokens": 4235, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2730317609999702, "nonpadding_tokens": 3736, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27358136199995897, "nonpadding_tokens": 8684, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27193210100006127, "nonpadding_tokens": 5621, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2731927469999391, "nonpadding_tokens": 7029, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2739579029999959, "nonpadding_tokens": 6586, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044403699996564683, "prefetch": {"data_wait_seconds": 0.007761973000654621, "consumed_batches": 822, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1372, "tokens": 108964876, "loss_tokens": 85619, "loss": 3.445375734358028, "lr": 0.0002997874466317624, "optimizer_lrs": [0.01998582977545083, 0.0002997874466317624], "gradient_norm": 2.2265071868896484, "seconds": 0.3329888389999951, "nonpadding_tokens_per_second": 257122.73197241084, "ranks": [{"compute_seconds": 0.3283566920000567, "nonpadding_tokens": 9727, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3265135110000301, "nonpadding_tokens": 4059, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3257094059999872, "nonpadding_tokens": 14406, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32647930899997846, "nonpadding_tokens": 9492, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3254321889999119, "nonpadding_tokens": 9860, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3293792959999564, "nonpadding_tokens": 5466, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.32494767299999694, "nonpadding_tokens": 19742, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3304993959999365, "nonpadding_tokens": 12867, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041063699995902425, "prefetch": {"data_wait_seconds": 0.007771135000737672, "consumed_batches": 823, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1373, "tokens": 109066303, "loss_tokens": 101427, "loss": 3.147506642708549, "lr": 0.0002997869618277461, "optimizer_lrs": [0.019985797455183075, 0.0002997869618277461], "gradient_norm": 1.0380715131759644, "seconds": 0.4462560389999908, "nonpadding_tokens_per_second": 227284.31917086526, "ranks": [{"compute_seconds": 0.442590000999985, "nonpadding_tokens": 20306, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4422405150000941, "nonpadding_tokens": 15394, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.43892902300001424, "nonpadding_tokens": 9057, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4427786159999414, "nonpadding_tokens": 18018, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4432075889999396, "nonpadding_tokens": 15960, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.44088609699997505, "nonpadding_tokens": 14032, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.443090467999923, "nonpadding_tokens": 4048, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4437413910000032, "nonpadding_tokens": 4612, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037483000005522626, "prefetch": {"data_wait_seconds": 0.00778137300069659, "consumed_batches": 824, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1374, "tokens": 109116879, "loss_tokens": 50576, "loss": 2.5549933515896868, "lr": 0.0002997867198767804, "optimizer_lrs": [0.019985781325118693, 0.0002997867198767804], "gradient_norm": 0.4701743423938751, "seconds": 0.3463136459999987, "nonpadding_tokens_per_second": 146041.02548127773, "ranks": [{"compute_seconds": 0.34396633700009716, "nonpadding_tokens": 15185, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33880973200007247, "nonpadding_tokens": 5722, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3411840960000063, "nonpadding_tokens": 6042, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.337865094000108, "nonpadding_tokens": 3819, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3356713740000714, "nonpadding_tokens": 3399, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3389167919999636, "nonpadding_tokens": 2276, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.34195967100004054, "nonpadding_tokens": 4212, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3365206569999373, "nonpadding_tokens": 9921, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004273160000138887, "prefetch": {"data_wait_seconds": 0.007799080000680192, "consumed_batches": 825, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1375, "tokens": 109175727, "loss_tokens": 58848, "loss": 2.5207911165205275, "lr": 0.000299786438180618, "optimizer_lrs": [0.019985762545374534, 0.000299786438180618], "gradient_norm": 0.8119300007820129, "seconds": 0.27139511899997615, "nonpadding_tokens_per_second": 216835.14507129058, "ranks": [{"compute_seconds": 0.26648608099992543, "nonpadding_tokens": 5434, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2673551430000316, "nonpadding_tokens": 5164, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2637773049999623, "nonpadding_tokens": 4192, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2680807650000361, "nonpadding_tokens": 4530, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2687269870000364, "nonpadding_tokens": 13765, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2660970529999531, "nonpadding_tokens": 8785, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26270467100005135, "nonpadding_tokens": 12747, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2690653189999921, "nonpadding_tokens": 4231, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003594749999820124, "prefetch": {"data_wait_seconds": 0.007809368000678774, "consumed_batches": 826, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1376, "tokens": 109221310, "loss_tokens": 45583, "loss": 2.8234489488954218, "lr": 0.00029978621985426356, "optimizer_lrs": [0.019985747990284238, 0.00029978621985426356], "gradient_norm": 0.5280868411064148, "seconds": 0.23326407000001836, "nonpadding_tokens_per_second": 195413.72145309995, "ranks": [{"compute_seconds": 0.22634291800000028, "nonpadding_tokens": 2304, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22705509699994764, "nonpadding_tokens": 6527, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22957170300003327, "nonpadding_tokens": 4969, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22992296699999315, "nonpadding_tokens": 6235, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2304725530000269, "nonpadding_tokens": 2608, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22758106000003409, "nonpadding_tokens": 7513, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23022686900003464, "nonpadding_tokens": 12520, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23085248500001398, "nonpadding_tokens": 2907, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036230900002465205, "prefetch": {"data_wait_seconds": 0.007818008000597843, "consumed_batches": 827, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1377, "tokens": 109286472, "loss_tokens": 65162, "loss": 2.7994822039685707, "lr": 0.00029978590755792216, "optimizer_lrs": [0.019985727170528145, 0.00029978590755792216], "gradient_norm": 0.3710082769393921, "seconds": 0.22086590100002468, "nonpadding_tokens_per_second": 295029.69768064254, "ranks": [{"compute_seconds": 0.21505874400008906, "nonpadding_tokens": 3197, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.216416797000079, "nonpadding_tokens": 5400, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21294894599998315, "nonpadding_tokens": 9394, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2139656010000408, "nonpadding_tokens": 6140, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21818881100000453, "nonpadding_tokens": 6743, "peak_allocated_bytes": 14979670528, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21553460799998447, "nonpadding_tokens": 15875, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2178842660000555, "nonpadding_tokens": 14121, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2185309359999792, "nonpadding_tokens": 4292, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037996399998974084, "prefetch": {"data_wait_seconds": 0.00782627700068872, "consumed_batches": 828, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1378, "tokens": 109355762, "loss_tokens": 69290, "loss": 2.704215976331361, "lr": 0.0002997855752278104, "optimizer_lrs": [0.01998570501518736, 0.0002997855752278104], "gradient_norm": 0.4356898367404938, "seconds": 0.3247888180000018, "nonpadding_tokens_per_second": 213338.6254695493, "ranks": [{"compute_seconds": 0.3219162849999293, "nonpadding_tokens": 7972, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31770646900008614, "nonpadding_tokens": 11464, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31802100599998084, "nonpadding_tokens": 8723, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3187075380000124, "nonpadding_tokens": 9524, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31679008100002193, "nonpadding_tokens": 12058, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3195842250000851, "nonpadding_tokens": 1454, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.32148159099995155, "nonpadding_tokens": 6264, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32237677600005554, "nonpadding_tokens": 11831, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035404599998400954, "prefetch": {"data_wait_seconds": 0.007833848000657895, "consumed_batches": 829, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1379, "tokens": 109451062, "loss_tokens": 95300, "loss": 2.8049222848898214, "lr": 0.00029978511772732697, "optimizer_lrs": [0.019985674515155132, 0.00029978511772732697], "gradient_norm": 0.3921453058719635, "seconds": 0.3328099979999024, "nonpadding_tokens_per_second": 286349.57054393523, "ranks": [{"compute_seconds": 0.3291289769999821, "nonpadding_tokens": 21200, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3265839450000385, "nonpadding_tokens": 23711, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3259565930000008, "nonpadding_tokens": 4108, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32625238299999637, "nonpadding_tokens": 6453, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.330457784000032, "nonpadding_tokens": 14609, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3275226789999124, "nonpadding_tokens": 5344, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3298651290000407, "nonpadding_tokens": 5027, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3303075930000432, "nonpadding_tokens": 14848, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003942600000073071, "prefetch": {"data_wait_seconds": 0.007841847000648272, "consumed_batches": 830, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1380, "tokens": 109499464, "loss_tokens": 48402, "loss": 2.4371213353787033, "lr": 0.0002997848851804648, "optimizer_lrs": [0.019985659012030987, 0.0002997848851804648], "gradient_norm": 0.4113762378692627, "seconds": 0.31464707299994643, "nonpadding_tokens_per_second": 153829.49391049394, "ranks": [{"compute_seconds": 0.3113500990000375, "nonpadding_tokens": 5601, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30785493500002303, "nonpadding_tokens": 2487, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3102525379999861, "nonpadding_tokens": 1818, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30595333099995514, "nonpadding_tokens": 10082, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3121014060000107, "nonpadding_tokens": 9994, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30872559800002364, "nonpadding_tokens": 2744, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31106155700001636, "nonpadding_tokens": 4585, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3055857389999801, "nonpadding_tokens": 11091, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004201149999971676, "prefetch": {"data_wait_seconds": 0.007866748000608936, "consumed_batches": 831, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1381, "tokens": 109550964, "loss_tokens": 51500, "loss": 2.960470266990291, "lr": 0.00029978463761131107, "optimizer_lrs": [0.01998564250742074, 0.00029978463761131107], "gradient_norm": 0.49779850244522095, "seconds": 0.3500486049999836, "nonpadding_tokens_per_second": 147122.42604138478, "ranks": [{"compute_seconds": 0.34414398299998084, "nonpadding_tokens": 5074, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3442423640000243, "nonpadding_tokens": 12569, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34688817400001426, "nonpadding_tokens": 7988, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3426743180000358, "nonpadding_tokens": 1510, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3415887429999884, "nonpadding_tokens": 10003, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34518279200005964, "nonpadding_tokens": 3682, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3476321549999284, "nonpadding_tokens": 5956, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34209763100000146, "nonpadding_tokens": 4718, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000435651999964648, "prefetch": {"data_wait_seconds": 0.007876330000684106, "consumed_batches": 832, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1382, "tokens": 109612215, "loss_tokens": 61251, "loss": 2.6635935025550603, "lr": 0.0002997843429822431, "optimizer_lrs": [0.019985622865482878, 0.0002997843429822431], "gradient_norm": 0.4296181797981262, "seconds": 0.23667987099997845, "nonpadding_tokens_per_second": 258792.60344875537, "ranks": [{"compute_seconds": 0.23217288900002586, "nonpadding_tokens": 6917, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.229645612000013, "nonpadding_tokens": 3774, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22795696399998633, "nonpadding_tokens": 9122, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2286168020000332, "nonpadding_tokens": 14129, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2332626979999759, "nonpadding_tokens": 6853, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23028907700006584, "nonpadding_tokens": 3757, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23433229800002664, "nonpadding_tokens": 4943, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2277221180000879, "nonpadding_tokens": 11756, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036005599997679383, "prefetch": {"data_wait_seconds": 0.007885204000629642, "consumed_batches": 833, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1383, "tokens": 109663948, "loss_tokens": 51733, "loss": 2.73211912608973, "lr": 0.00029978409397982234, "optimizer_lrs": [0.01998560626532149, 0.00029978409397982234], "gradient_norm": 0.4278797209262848, "seconds": 0.42773510000006354, "nonpadding_tokens_per_second": 120946.35207630217, "ranks": [{"compute_seconds": 0.4251203069999292, "nonpadding_tokens": 8295, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4225447669999767, "nonpadding_tokens": 7526, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.42425339800001893, "nonpadding_tokens": 3398, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4245703889999959, "nonpadding_tokens": 6428, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4187324309999667, "nonpadding_tokens": 9861, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4216133260000561, "nonpadding_tokens": 4077, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.4245629509999844, "nonpadding_tokens": 7589, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.42528712399996493, "nonpadding_tokens": 4559, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004308870001068499, "prefetch": {"data_wait_seconds": 0.007894292000585779, "consumed_batches": 834, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1384, "tokens": 109726658, "loss_tokens": 62710, "loss": 2.6436553181310796, "lr": 0.0002997837919501865, "optimizer_lrs": [0.019985586130012434, 0.0002997837919501865], "gradient_norm": 0.4671696722507477, "seconds": 0.2774582580000242, "nonpadding_tokens_per_second": 226015.97967213695, "ranks": [{"compute_seconds": 0.2746443690000433, "nonpadding_tokens": 10923, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2717238489999545, "nonpadding_tokens": 9631, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2717530690000558, "nonpadding_tokens": 4685, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2724527880000096, "nonpadding_tokens": 5001, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2706825340000023, "nonpadding_tokens": 9734, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2728354919998992, "nonpadding_tokens": 4019, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.27511192199995094, "nonpadding_tokens": 10474, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27167399799998293, "nonpadding_tokens": 8243, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003732590000709024, "prefetch": {"data_wait_seconds": 0.00790421800066099, "consumed_batches": 835, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1385, "tokens": 109767686, "loss_tokens": 41028, "loss": 2.446251264380423, "lr": 0.0002997835942332339, "optimizer_lrs": [0.01998557294888226, 0.0002997835942332339], "gradient_norm": 0.44389474391937256, "seconds": 0.27422084499994526, "nonpadding_tokens_per_second": 149616.6347237687, "ranks": [{"compute_seconds": 0.27052777599999445, "nonpadding_tokens": 5561, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.267780413999958, "nonpadding_tokens": 2196, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27038490200004617, "nonpadding_tokens": 5921, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2708584150000206, "nonpadding_tokens": 6083, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26923317700004645, "nonpadding_tokens": 1487, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27153268600000047, "nonpadding_tokens": 3397, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2709005579999939, "nonpadding_tokens": 6475, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2718528550000201, "nonpadding_tokens": 9908, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004130869999698916, "prefetch": {"data_wait_seconds": 0.007912841000646154, "consumed_batches": 836, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1386, "tokens": 109908918, "loss_tokens": 141232, "loss": 2.271968286224085, "lr": 0.00029978291293547354, "optimizer_lrs": [0.01998552752903157, 0.00029978291293547354], "gradient_norm": 0.4711374342441559, "seconds": 0.3284494909999012, "nonpadding_tokens_per_second": 429996.0994612791, "ranks": [{"compute_seconds": 0.3260620020000715, "nonpadding_tokens": 17155, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.324174706000008, "nonpadding_tokens": 25626, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32460072999992917, "nonpadding_tokens": 17502, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3254645929999924, "nonpadding_tokens": 24846, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3257895680000047, "nonpadding_tokens": 20513, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3231463060000124, "nonpadding_tokens": 9785, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31991326699994715, "nonpadding_tokens": 11571, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32126457599997593, "nonpadding_tokens": 14234, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003848139999718114, "prefetch": {"data_wait_seconds": 0.00792219700072394, "consumed_batches": 837, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1387, "tokens": 109996758, "loss_tokens": 87840, "loss": 2.4082614697176683, "lr": 0.00029978248865902894, "optimizer_lrs": [0.019985499243935265, 0.00029978248865902894], "gradient_norm": 0.4096177816390991, "seconds": 0.31596687199998996, "nonpadding_tokens_per_second": 278003.8281988081, "ranks": [{"compute_seconds": 0.30201612600001226, "nonpadding_tokens": 14402, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30761773699998685, "nonpadding_tokens": 3225, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3096081599999252, "nonpadding_tokens": 13300, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30912093699998877, "nonpadding_tokens": 11396, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3111082309999347, "nonpadding_tokens": 15761, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3081182969999645, "nonpadding_tokens": 20395, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3109086560000378, "nonpadding_tokens": 2683, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31122568500006764, "nonpadding_tokens": 6678, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003897569999935513, "prefetch": {"data_wait_seconds": 0.007932354000743658, "consumed_batches": 838, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1388, "tokens": 110049443, "loss_tokens": 52685, "loss": 2.738543643826516, "lr": 0.00029978223398637716, "optimizer_lrs": [0.01998548226575848, 0.00029978223398637716], "gradient_norm": 0.3966340124607086, "seconds": 0.3105924909999658, "nonpadding_tokens_per_second": 169627.4105995879, "ranks": [{"compute_seconds": 0.30715803999999025, "nonpadding_tokens": 7129, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30831587699992724, "nonpadding_tokens": 3484, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3046176980000155, "nonpadding_tokens": 3278, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3049901519999594, "nonpadding_tokens": 5589, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3039642119999826, "nonpadding_tokens": 5211, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3051659060000702, "nonpadding_tokens": 11492, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.30737259099998937, "nonpadding_tokens": 8522, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3078514640000094, "nonpadding_tokens": 7980, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033786699998472614, "prefetch": {"data_wait_seconds": 0.00794087900074203, "consumed_batches": 839, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1389, "tokens": 110110242, "loss_tokens": 60799, "loss": 2.677225108554417, "lr": 0.0002997819399066278, "optimizer_lrs": [0.019985462660441853, 0.0002997819399066278], "gradient_norm": 0.3930480480194092, "seconds": 0.24357745999998315, "nonpadding_tokens_per_second": 249608.48183573392, "ranks": [{"compute_seconds": 0.2410241820000465, "nonpadding_tokens": 2217, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23758145199997216, "nonpadding_tokens": 8282, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.237914657000033, "nonpadding_tokens": 11306, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2383444459999282, "nonpadding_tokens": 9255, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24120756299998902, "nonpadding_tokens": 11263, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23812233400008154, "nonpadding_tokens": 4996, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23694299600003887, "nonpadding_tokens": 6348, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23740571799999088, "nonpadding_tokens": 7132, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003754120000394323, "prefetch": {"data_wait_seconds": 0.00794890700076678, "consumed_batches": 840, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1390, "tokens": 110172616, "loss_tokens": 62374, "loss": 2.698128927918684, "lr": 0.00029978163800268844, "optimizer_lrs": [0.019985442533512563, 0.00029978163800268844], "gradient_norm": 0.3938305079936981, "seconds": 0.2727469740000288, "nonpadding_tokens_per_second": 228688.14669230176, "ranks": [{"compute_seconds": 0.27039007299993045, "nonpadding_tokens": 3644, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27034009999999853, "nonpadding_tokens": 3821, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26900140599991573, "nonpadding_tokens": 8237, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2676846839999598, "nonpadding_tokens": 10479, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2702291340000329, "nonpadding_tokens": 13315, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26901774200007367, "nonpadding_tokens": 6373, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2701971509999339, "nonpadding_tokens": 13286, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27040742999997747, "nonpadding_tokens": 3219, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003953259999889269, "prefetch": {"data_wait_seconds": 0.007957567000815402, "consumed_batches": 841, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1391, "tokens": 110223327, "loss_tokens": 50711, "loss": 2.7020052971741833, "lr": 0.00029978139239644823, "optimizer_lrs": [0.019985426159763217, 0.00029978139239644823], "gradient_norm": 0.4072178304195404, "seconds": 0.3248340830000416, "nonpadding_tokens_per_second": 156113.54427975314, "ranks": [{"compute_seconds": 0.32180941199999324, "nonpadding_tokens": 3545, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3175739370000201, "nonpadding_tokens": 6676, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3181245220000619, "nonpadding_tokens": 10404, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3212536659999614, "nonpadding_tokens": 8182, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31722221899997294, "nonpadding_tokens": 10755, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31893522899997606, "nonpadding_tokens": 2271, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31662689400002364, "nonpadding_tokens": 5389, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.322397376999902, "nonpadding_tokens": 3489, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004355460000624589, "prefetch": {"data_wait_seconds": 0.007975750000767334, "consumed_batches": 842, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1392, "tokens": 110307969, "loss_tokens": 84642, "loss": 2.7172000897899387, "lr": 0.00029978098214651346, "optimizer_lrs": [0.019985398809767563, 0.00029978098214651346], "gradient_norm": 0.3925826847553253, "seconds": 0.26015889400002834, "nonpadding_tokens_per_second": 325347.32408568275, "ranks": [{"compute_seconds": 0.24646998299999723, "nonpadding_tokens": 3007, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25325653600009446, "nonpadding_tokens": 13108, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2544164529999762, "nonpadding_tokens": 17626, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2564476119999881, "nonpadding_tokens": 14251, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25687255100001494, "nonpadding_tokens": 19804, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2543073379999896, "nonpadding_tokens": 5238, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2570362630000318, "nonpadding_tokens": 10070, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2577147200000809, "nonpadding_tokens": 1538, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000410255000019788, "prefetch": {"data_wait_seconds": 0.007984787000808069, "consumed_batches": 843, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1393, "tokens": 110360058, "loss_tokens": 52089, "loss": 2.7346302722263816, "lr": 0.00029978072948617706, "optimizer_lrs": [0.01998538196574514, 0.00029978072948617706], "gradient_norm": 0.411436527967453, "seconds": 0.31195472100000643, "nonpadding_tokens_per_second": 166976.15549148535, "ranks": [{"compute_seconds": 0.30831473600005666, "nonpadding_tokens": 11351, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3052841390000367, "nonpadding_tokens": 6482, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30495956299989757, "nonpadding_tokens": 3277, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3076648919999343, "nonpadding_tokens": 10843, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3094001900000194, "nonpadding_tokens": 3727, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.303737907000027, "nonpadding_tokens": 3725, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.30772408200004975, "nonpadding_tokens": 8375, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3085754439999846, "nonpadding_tokens": 4309, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034862000006796734, "prefetch": {"data_wait_seconds": 0.0079941830008039, "consumed_batches": 844, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1394, "tokens": 110420220, "loss_tokens": 60162, "loss": 2.962463847611449, "lr": 0.0002997804374862309, "optimizer_lrs": [0.019985362499082063, 0.0002997804374862309], "gradient_norm": 0.3978300094604492, "seconds": 0.24271414599991203, "nonpadding_tokens_per_second": 247871.83191218614, "ranks": [{"compute_seconds": 0.2400484999999435, "nonpadding_tokens": 3539, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2372124939998912, "nonpadding_tokens": 2238, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23921533899999758, "nonpadding_tokens": 6378, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2350133670000787, "nonpadding_tokens": 9613, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24013746100001754, "nonpadding_tokens": 4347, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23613961000000927, "nonpadding_tokens": 11737, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23307903700003862, "nonpadding_tokens": 18476, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2341264429999228, "nonpadding_tokens": 3834, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004089709999561819, "prefetch": {"data_wait_seconds": 0.008004332000794534, "consumed_batches": 845, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1395, "tokens": 110500763, "loss_tokens": 80543, "loss": 2.555416633040736, "lr": 0.0002997800462619239, "optimizer_lrs": [0.019985336417461596, 0.0002997800462619239], "gradient_norm": 0.44491562247276306, "seconds": 0.31846742899995206, "nonpadding_tokens_per_second": 252908.12392626854, "ranks": [{"compute_seconds": 0.3157624869999154, "nonpadding_tokens": 7570, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3126557140000159, "nonpadding_tokens": 9605, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3105101139999533, "nonpadding_tokens": 12831, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31100338299995656, "nonpadding_tokens": 2606, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3160084300000108, "nonpadding_tokens": 19213, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31384571099999903, "nonpadding_tokens": 12914, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3160699850000128, "nonpadding_tokens": 2765, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31039964700005385, "nonpadding_tokens": 13039, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004280929999822547, "prefetch": {"data_wait_seconds": 0.008013826000777158, "consumed_batches": 846, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1396, "tokens": 110543888, "loss_tokens": 43125, "loss": 2.796119927536232, "lr": 0.0002997798366463371, "optimizer_lrs": [0.01998532244308914, 0.0002997798366463371], "gradient_norm": 0.5164533853530884, "seconds": 0.27722776400003113, "nonpadding_tokens_per_second": 155558.0125805695, "ranks": [{"compute_seconds": 0.274238940000032, "nonpadding_tokens": 5378, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27073730799997975, "nonpadding_tokens": 9488, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27360717600004136, "nonpadding_tokens": 2229, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27101484899992556, "nonpadding_tokens": 6240, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26934100699998, "nonpadding_tokens": 7393, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27146743600007994, "nonpadding_tokens": 5553, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26988919300004, "nonpadding_tokens": 4005, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2749536999999691, "nonpadding_tokens": 2839, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004006599999684113, "prefetch": {"data_wait_seconds": 0.00802356600077303, "consumed_batches": 847, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1397, "tokens": 110594455, "loss_tokens": 50567, "loss": 2.438314204421856, "lr": 0.00029977959073072434, "optimizer_lrs": [0.019985306048714957, 0.00029977959073072434], "gradient_norm": 0.47518888115882874, "seconds": 0.24599339100007, "nonpadding_tokens_per_second": 205562.4331792947, "ranks": [{"compute_seconds": 0.24372777200005658, "nonpadding_tokens": 1960, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24004608100005953, "nonpadding_tokens": 3737, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24241094999990764, "nonpadding_tokens": 2589, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24309604999996282, "nonpadding_tokens": 5746, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23957526599997436, "nonpadding_tokens": 7164, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24183157499999197, "nonpadding_tokens": 8822, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23885525200000757, "nonpadding_tokens": 9770, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24015534500006197, "nonpadding_tokens": 10779, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004080479999402087, "prefetch": {"data_wait_seconds": 0.008030583000802949, "consumed_batches": 848, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1398, "tokens": 110641762, "loss_tokens": 47307, "loss": 2.6551007580801995, "lr": 0.00029977936054485294, "optimizer_lrs": [0.019985290702990197, 0.00029977936054485294], "gradient_norm": 0.426545113325119, "seconds": 0.27996698199990533, "nonpadding_tokens_per_second": 168973.49702478843, "ranks": [{"compute_seconds": 0.27620203899994067, "nonpadding_tokens": 2290, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.274935934000041, "nonpadding_tokens": 4694, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2758420650000062, "nonpadding_tokens": 4039, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27248715999996875, "nonpadding_tokens": 4891, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27113594800005103, "nonpadding_tokens": 6218, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27396383000007063, "nonpadding_tokens": 8262, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2708365140000524, "nonpadding_tokens": 8318, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.277550738000059, "nonpadding_tokens": 8595, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003433449999192817, "prefetch": {"data_wait_seconds": 0.008038467000758942, "consumed_batches": 849, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1399, "tokens": 110715546, "loss_tokens": 73784, "loss": 2.7978300681719612, "lr": 0.0002997790012878792, "optimizer_lrs": [0.019985266752525283, 0.0002997790012878792], "gradient_norm": 0.40852439403533936, "seconds": 0.4170810260000053, "nonpadding_tokens_per_second": 176905.67395889896, "ranks": [{"compute_seconds": 0.4139332720000084, "nonpadding_tokens": 16983, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4142772899999727, "nonpadding_tokens": 3444, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41372365099994113, "nonpadding_tokens": 10976, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4121000100000174, "nonpadding_tokens": 2204, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.4103468430000703, "nonpadding_tokens": 7664, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.413500302999978, "nonpadding_tokens": 11398, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.41479257100002087, "nonpadding_tokens": 8213, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4106229170000688, "nonpadding_tokens": 12902, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038032599991311145, "prefetch": {"data_wait_seconds": 0.008046908000736153, "consumed_batches": 850, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1400, "tokens": 110780982, "loss_tokens": 65436, "loss": 2.652892473179901, "lr": 0.0002997786824333046, "optimizer_lrs": [0.019985245495553642, 0.0002997786824333046], "gradient_norm": 0.38000044226646423, "seconds": 0.2895693840000604, "nonpadding_tokens_per_second": 225976.92855535567, "ranks": [{"compute_seconds": 0.28720880900004886, "nonpadding_tokens": 9114, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28563162900002226, "nonpadding_tokens": 5567, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28633209700001316, "nonpadding_tokens": 12887, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28289890900009595, "nonpadding_tokens": 8528, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2816264890000184, "nonpadding_tokens": 6953, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28429307199996856, "nonpadding_tokens": 13609, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28669214399997145, "nonpadding_tokens": 1281, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28221787199993287, "nonpadding_tokens": 7497, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003903319999380983, "prefetch": {"data_wait_seconds": 0.008056952000700335, "consumed_batches": 851, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1401, "tokens": 110822469, "loss_tokens": 41487, "loss": 2.6563343637766046, "lr": 0.00029977848015773356, "optimizer_lrs": [0.019985232010515573, 0.00029977848015773356], "gradient_norm": 0.4008479714393616, "seconds": 0.23906458199996905, "nonpadding_tokens_per_second": 173538.88080336957, "ranks": [{"compute_seconds": 0.2367508889999499, "nonpadding_tokens": 3972, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2323485620000838, "nonpadding_tokens": 2801, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23504555300007723, "nonpadding_tokens": 8047, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23236920099998315, "nonpadding_tokens": 5020, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23602913100000933, "nonpadding_tokens": 3289, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23326254199992036, "nonpadding_tokens": 10944, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23579234399994675, "nonpadding_tokens": 5465, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23655180200000814, "nonpadding_tokens": 1949, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004219489999286452, "prefetch": {"data_wait_seconds": 0.008066773000749095, "consumed_batches": 852, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1402, "tokens": 110898033, "loss_tokens": 75564, "loss": 2.3982338233153353, "lr": 0.0002997781114978613, "optimizer_lrs": [0.019985207433190755, 0.0002997781114978613], "gradient_norm": 0.742050051689148, "seconds": 0.3121715900000481, "nonpadding_tokens_per_second": 242059.1828999825, "ranks": [{"compute_seconds": 0.30958846700002596, "nonpadding_tokens": 2343, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3089075979999052, "nonpadding_tokens": 17427, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30978075399991667, "nonpadding_tokens": 4140, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3090499869999803, "nonpadding_tokens": 17229, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3094406989999925, "nonpadding_tokens": 2951, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30658693800000947, "nonpadding_tokens": 11691, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.30500903899996956, "nonpadding_tokens": 17733, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30585412199991424, "nonpadding_tokens": 2050, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003947249999782798, "prefetch": {"data_wait_seconds": 0.008075838000721092, "consumed_batches": 853, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1403, "tokens": 110994505, "loss_tokens": 96472, "loss": 2.341072571575172, "lr": 0.00029977764038743545, "optimizer_lrs": [0.01998517602582903, 0.00029977764038743545], "gradient_norm": 0.9094254374504089, "seconds": 0.3319331569999804, "nonpadding_tokens_per_second": 290636.8284262898, "ranks": [{"compute_seconds": 0.32908278399997926, "nonpadding_tokens": 4424, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3256476850000354, "nonpadding_tokens": 6974, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32822653300002, "nonpadding_tokens": 13577, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3287794949999352, "nonpadding_tokens": 17622, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3250825590000659, "nonpadding_tokens": 17844, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3281664029999547, "nonpadding_tokens": 16444, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3252210959999502, "nonpadding_tokens": 10054, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3296082809999916, "nonpadding_tokens": 9533, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040995000006205373, "prefetch": {"data_wait_seconds": 0.008086008000717015, "consumed_batches": 854, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1404, "tokens": 111073100, "loss_tokens": 78595, "loss": 2.8808808209809786, "lr": 0.00029977725620837584, "optimizer_lrs": [0.019985150413891723, 0.00029977725620837584], "gradient_norm": 0.42836588621139526, "seconds": 0.2905113670000219, "nonpadding_tokens_per_second": 270540.1885358726, "ranks": [{"compute_seconds": 0.28820300999996107, "nonpadding_tokens": 3702, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28455300400003125, "nonpadding_tokens": 13531, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2826050089998944, "nonpadding_tokens": 2002, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28325389999997697, "nonpadding_tokens": 10287, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28169504599998163, "nonpadding_tokens": 10320, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2842056519999687, "nonpadding_tokens": 8319, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28121188800002983, "nonpadding_tokens": 12972, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28224635900005524, "nonpadding_tokens": 17462, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003474380000625388, "prefetch": {"data_wait_seconds": 0.0080952870008133, "consumed_batches": 855, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1405, "tokens": 111131131, "loss_tokens": 58031, "loss": 2.8328619617101203, "lr": 0.00029977697233527276, "optimizer_lrs": [0.019985131489018187, 0.00029977697233527276], "gradient_norm": 0.4893612563610077, "seconds": 0.2685814289999371, "nonpadding_tokens_per_second": 216064.827028728, "ranks": [{"compute_seconds": 0.2642543399999795, "nonpadding_tokens": 4385, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2617299199999934, "nonpadding_tokens": 1644, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2643615189999764, "nonpadding_tokens": 1839, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26113829300004454, "nonpadding_tokens": 9977, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2598624500000142, "nonpadding_tokens": 6183, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2624976820000029, "nonpadding_tokens": 10628, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2651612309999791, "nonpadding_tokens": 17878, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26613594200000534, "nonpadding_tokens": 5497, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041752999993605044, "prefetch": {"data_wait_seconds": 0.00810482200081708, "consumed_batches": 856, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1406, "tokens": 111198567, "loss_tokens": 67436, "loss": 2.3717360256391244, "lr": 0.0002997766422283606, "optimizer_lrs": [0.019985109481890705, 0.0002997766422283606], "gradient_norm": 0.4590141773223877, "seconds": 0.22252429800005302, "nonpadding_tokens_per_second": 303050.05163968174, "ranks": [{"compute_seconds": 0.21809616200005166, "nonpadding_tokens": 2958, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21588110800007598, "nonpadding_tokens": 4610, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21492251999995915, "nonpadding_tokens": 9888, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21886183100002654, "nonpadding_tokens": 12766, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.22017483399997673, "nonpadding_tokens": 9630, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21694542900002034, "nonpadding_tokens": 10993, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.21919413499995244, "nonpadding_tokens": 11711, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21459431099992798, "nonpadding_tokens": 4880, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003388789999689834, "prefetch": {"data_wait_seconds": 0.0081118350008893, "consumed_batches": 857, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1407, "tokens": 111264589, "loss_tokens": 66022, "loss": 2.6577092673654237, "lr": 0.0002997763188068422, "optimizer_lrs": [0.01998508792045615, 0.0002997763188068422], "gradient_norm": 0.3780006468296051, "seconds": 0.30740360199990846, "nonpadding_tokens_per_second": 214773.02012882615, "ranks": [{"compute_seconds": 0.30500528200002464, "nonpadding_tokens": 2597, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3029850290000695, "nonpadding_tokens": 4326, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3034780479999881, "nonpadding_tokens": 2141, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30366146799997296, "nonpadding_tokens": 5067, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2990581309999243, "nonpadding_tokens": 13171, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3014506759999449, "nonpadding_tokens": 10295, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.30449749599995357, "nonpadding_tokens": 7833, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.299709302999986, "nonpadding_tokens": 20592, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003845989999717858, "prefetch": {"data_wait_seconds": 0.008120277000855936, "consumed_batches": 858, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1408, "tokens": 111351360, "loss_tokens": 86771, "loss": 2.6888361318873817, "lr": 0.0002997758933867884, "optimizer_lrs": [0.019985059559119226, 0.0002997758933867884], "gradient_norm": 0.4000682234764099, "seconds": 0.32077282799991735, "nonpadding_tokens_per_second": 270506.07914964156, "ranks": [{"compute_seconds": 0.3157934390000037, "nonpadding_tokens": 25605, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3183338430000049, "nonpadding_tokens": 12117, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.315494005000005, "nonpadding_tokens": 12224, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3180787589999454, "nonpadding_tokens": 6699, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31840960199997426, "nonpadding_tokens": 9333, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3167993210000759, "nonpadding_tokens": 10553, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31791386499992313, "nonpadding_tokens": 6599, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3147457329999952, "nonpadding_tokens": 3641, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003954239999757192, "prefetch": {"data_wait_seconds": 0.008129740000825336, "consumed_batches": 859, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1409, "tokens": 111405813, "loss_tokens": 54453, "loss": 2.4234199102896077, "lr": 0.00029977562620889426, "optimizer_lrs": [0.019985041747259617, 0.00029977562620889426], "gradient_norm": 0.4240371286869049, "seconds": 0.32906497500005116, "nonpadding_tokens_per_second": 165477.9576586403, "ranks": [{"compute_seconds": 0.2863907350000545, "nonpadding_tokens": 11611, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2873435239999935, "nonpadding_tokens": 10537, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29037975799997184, "nonpadding_tokens": 2585, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2879653210000015, "nonpadding_tokens": 9886, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28608856300002117, "nonpadding_tokens": 9592, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2884614160000183, "nonpadding_tokens": 4682, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.28584625600001345, "nonpadding_tokens": 3738, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2866954439999745, "nonpadding_tokens": 1822, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.025216572999966047, "prefetch": {"data_wait_seconds": 0.008138374000807858, "consumed_batches": 860, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1410, "tokens": 111445747, "loss_tokens": 39934, "loss": 2.4712707608178492, "lr": 0.00029977543016852386, "optimizer_lrs": [0.01998502867790159, 0.00029977543016852386], "gradient_norm": 2.601722002029419, "seconds": 0.27151830200000404, "nonpadding_tokens_per_second": 147076.64163279647, "ranks": [{"compute_seconds": 0.26368670900001234, "nonpadding_tokens": 11567, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2659671700000672, "nonpadding_tokens": 1294, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2680510840000352, "nonpadding_tokens": 2089, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26901121599996713, "nonpadding_tokens": 4627, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2685478900000362, "nonpadding_tokens": 6798, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2658492240000214, "nonpadding_tokens": 2868, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26829033900003196, "nonpadding_tokens": 5316, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2628655900000467, "nonpadding_tokens": 5375, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037107399998603796, "prefetch": {"data_wait_seconds": 0.008148123000822238, "consumed_batches": 861, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1411, "tokens": 111516772, "loss_tokens": 71025, "loss": 2.8256252199929603, "lr": 0.0002997750812876898, "optimizer_lrs": [0.019985005419179323, 0.0002997750812876898], "gradient_norm": 0.4057559072971344, "seconds": 0.25205767600004947, "nonpadding_tokens_per_second": 281780.7460860111, "ranks": [{"compute_seconds": 0.24652035799999794, "nonpadding_tokens": 1809, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2485833550000507, "nonpadding_tokens": 6585, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2452824630000805, "nonpadding_tokens": 2889, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2491620260000218, "nonpadding_tokens": 19353, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24925008000002435, "nonpadding_tokens": 6179, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24719303099993795, "nonpadding_tokens": 4300, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2440888679999489, "nonpadding_tokens": 17326, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2496617229999174, "nonpadding_tokens": 12584, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038510899992161285, "prefetch": {"data_wait_seconds": 0.008155594000825772, "consumed_batches": 862, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1412, "tokens": 111559376, "loss_tokens": 42604, "loss": 2.7786794520467564, "lr": 0.0002997748718833967, "optimizer_lrs": [0.019984991458893115, 0.0002997748718833967], "gradient_norm": 0.4480742812156677, "seconds": 0.364068331999988, "nonpadding_tokens_per_second": 117021.9880591026, "ranks": [{"compute_seconds": 0.361201147000088, "nonpadding_tokens": 12719, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36184900500006734, "nonpadding_tokens": 1677, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35924065699998664, "nonpadding_tokens": 1250, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3593908149999834, "nonpadding_tokens": 5561, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3602310150000676, "nonpadding_tokens": 2983, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3607922339999732, "nonpadding_tokens": 11223, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3580641959999866, "nonpadding_tokens": 2629, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35894126000005144, "nonpadding_tokens": 4562, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003805789999660192, "prefetch": {"data_wait_seconds": 0.00816424600084531, "consumed_batches": 863, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1413, "tokens": 111623385, "loss_tokens": 64009, "loss": 2.4932968410692244, "lr": 0.00029977455708769004, "optimizer_lrs": [0.019984970472512673, 0.00029977455708769004], "gradient_norm": 0.4456941783428192, "seconds": 0.27362901900005454, "nonpadding_tokens_per_second": 233926.21233637226, "ranks": [{"compute_seconds": 0.24655798000003415, "nonpadding_tokens": 9898, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.258000538000033, "nonpadding_tokens": 2629, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25506869700006973, "nonpadding_tokens": 6123, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2562162259999923, "nonpadding_tokens": 13643, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2540906990000167, "nonpadding_tokens": 3287, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2534682560000192, "nonpadding_tokens": 11608, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2541761070000348, "nonpadding_tokens": 8554, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25480210399996395, "nonpadding_tokens": 8267, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000565158999961568, "prefetch": {"data_wait_seconds": 0.008173047000880729, "consumed_batches": 864, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1414, "tokens": 111673868, "loss_tokens": 50483, "loss": 2.3323546107105364, "lr": 0.0002997743086577313, "optimizer_lrs": [0.019984953910515423, 0.0002997743086577313], "gradient_norm": 0.38967686891555786, "seconds": 0.27117455800009793, "nonpadding_tokens_per_second": 186164.21972736015, "ranks": [{"compute_seconds": 0.2652050039999949, "nonpadding_tokens": 6769, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26557134600000154, "nonpadding_tokens": 7879, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2674414960000604, "nonpadding_tokens": 6254, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26837323899997045, "nonpadding_tokens": 3090, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2610334639999792, "nonpadding_tokens": 4570, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26616766500001177, "nonpadding_tokens": 5754, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26079289300002984, "nonpadding_tokens": 5931, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.268553967999992, "nonpadding_tokens": 10236, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004859480000050098, "prefetch": {"data_wait_seconds": 0.008184062000921699, "consumed_batches": 865, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1415, "tokens": 111721508, "loss_tokens": 47640, "loss": 2.6839999540827035, "lr": 0.00029977407409299047, "optimizer_lrs": [0.019984938272866033, 0.00029977407409299047], "gradient_norm": 0.4418567419052124, "seconds": 0.31486484700008077, "nonpadding_tokens_per_second": 151303.0128764669, "ranks": [{"compute_seconds": 0.3101985360000299, "nonpadding_tokens": 5228, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30882244500003253, "nonpadding_tokens": 2684, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3068736640000225, "nonpadding_tokens": 11788, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31223015799992027, "nonpadding_tokens": 4104, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3061582490000774, "nonpadding_tokens": 9121, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30945399699999143, "nonpadding_tokens": 6070, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31250852000005125, "nonpadding_tokens": 3711, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30644835700002204, "nonpadding_tokens": 4934, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042433900000560243, "prefetch": {"data_wait_seconds": 0.008194881000875398, "consumed_batches": 866, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1416, "tokens": 111783345, "loss_tokens": 61837, "loss": 2.6239473333117713, "lr": 0.00029977376944503143, "optimizer_lrs": [0.019984917963002096, 0.00029977376944503143], "gradient_norm": 0.44541895389556885, "seconds": 0.23293897299993205, "nonpadding_tokens_per_second": 265464.37980568426, "ranks": [{"compute_seconds": 0.22895989300002384, "nonpadding_tokens": 4353, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22675839199996517, "nonpadding_tokens": 10350, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22613071399996443, "nonpadding_tokens": 9501, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22611517999996522, "nonpadding_tokens": 10276, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2306072019999874, "nonpadding_tokens": 5496, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2276262720000659, "nonpadding_tokens": 1760, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.23019057099998008, "nonpadding_tokens": 1406, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22571151800002554, "nonpadding_tokens": 18695, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003904210000200692, "prefetch": {"data_wait_seconds": 0.008204825000873939, "consumed_batches": 867, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1417, "tokens": 111843584, "loss_tokens": 60239, "loss": 2.703056263384186, "lr": 0.0002997734724726176, "optimizer_lrs": [0.019984898164841173, 0.0002997734724726176], "gradient_norm": 0.4082268476486206, "seconds": 0.29012711600000785, "nonpadding_tokens_per_second": 207629.67912312743, "ranks": [{"compute_seconds": 0.28348950300005527, "nonpadding_tokens": 2754, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2844183020000628, "nonpadding_tokens": 2073, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2872256059999927, "nonpadding_tokens": 12550, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28728992499998185, "nonpadding_tokens": 2165, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2875207729999829, "nonpadding_tokens": 6295, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28570920699996805, "nonpadding_tokens": 12843, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2871552290000636, "nonpadding_tokens": 10345, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2810283269999445, "nonpadding_tokens": 11214, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004066779999902792, "prefetch": {"data_wait_seconds": 0.008214149000949078, "consumed_batches": 868, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1418, "tokens": 111921886, "loss_tokens": 78302, "loss": 2.738436198947664, "lr": 0.0002997730861604545, "optimizer_lrs": [0.01998487241069697, 0.0002997730861604545], "gradient_norm": 0.37914374470710754, "seconds": 0.31433439299996735, "nonpadding_tokens_per_second": 249104.14432444284, "ranks": [{"compute_seconds": 0.30670533700003944, "nonpadding_tokens": 14572, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30734613300001, "nonpadding_tokens": 22367, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3100032290000172, "nonpadding_tokens": 3621, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31097579400000086, "nonpadding_tokens": 10804, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3111166769999727, "nonpadding_tokens": 13109, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30703066900002796, "nonpadding_tokens": 5477, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.31096768700001576, "nonpadding_tokens": 6681, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30457349999994676, "nonpadding_tokens": 1671, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004365690000440736, "prefetch": {"data_wait_seconds": 0.008224301000907985, "consumed_batches": 869, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1419, "tokens": 111985705, "loss_tokens": 63819, "loss": 2.5300153069618765, "lr": 0.000299772771058649, "optimizer_lrs": [0.019984851403909935, 0.000299772771058649], "gradient_norm": 0.412812203168869, "seconds": 0.3296653029999561, "nonpadding_tokens_per_second": 193587.2517345524, "ranks": [{"compute_seconds": 0.3252621870000212, "nonpadding_tokens": 2725, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3236450340000374, "nonpadding_tokens": 12815, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3257745959999738, "nonpadding_tokens": 16589, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3212650759999178, "nonpadding_tokens": 2213, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31988116900004115, "nonpadding_tokens": 4015, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32220402699999795, "nonpadding_tokens": 5400, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3271027900000263, "nonpadding_tokens": 8761, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32704023100006907, "nonpadding_tokens": 11301, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004076629999190118, "prefetch": {"data_wait_seconds": 0.008234148000909158, "consumed_batches": 870, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1420, "tokens": 112036736, "loss_tokens": 51031, "loss": 2.5083467953302896, "lr": 0.0002997725189395173, "optimizer_lrs": [0.01998483459596782, 0.0002997725189395173], "gradient_norm": 0.3933301270008087, "seconds": 0.3036681490000319, "nonpadding_tokens_per_second": 168048.57594727408, "ranks": [{"compute_seconds": 0.29600078400005714, "nonpadding_tokens": 5409, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29581827300000896, "nonpadding_tokens": 17270, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2992945780000582, "nonpadding_tokens": 4525, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3010431919999519, "nonpadding_tokens": 1601, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2988014820000444, "nonpadding_tokens": 6133, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29604214099992987, "nonpadding_tokens": 3497, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.29838059600001543, "nonpadding_tokens": 9210, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2923448499999495, "nonpadding_tokens": 3386, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038249900001119386, "prefetch": {"data_wait_seconds": 0.00824376500095525, "consumed_batches": 871, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1421, "tokens": 112106875, "loss_tokens": 70139, "loss": 2.6269668533198365, "lr": 0.00029977217218922134, "optimizer_lrs": [0.019984811479281425, 0.00029977217218922134], "gradient_norm": 0.38782426714897156, "seconds": 0.2955681140000479, "nonpadding_tokens_per_second": 237302.32280735343, "ranks": [{"compute_seconds": 0.2919920979999233, "nonpadding_tokens": 11288, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29013207399998464, "nonpadding_tokens": 6543, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29027129999997214, "nonpadding_tokens": 5776, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29299987900003543, "nonpadding_tokens": 7379, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29314375599994946, "nonpadding_tokens": 4173, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2919665519999626, "nonpadding_tokens": 11897, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2895270530000289, "nonpadding_tokens": 10208, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28990468500001043, "nonpadding_tokens": 12875, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042171999996298837, "prefetch": {"data_wait_seconds": 0.008253749000985522, "consumed_batches": 872, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1422, "tokens": 112150788, "loss_tokens": 43913, "loss": 2.4838501696536333, "lr": 0.00029977195495963307, "optimizer_lrs": [0.019984796997308873, 0.00029977195495963307], "gradient_norm": 0.4209425747394562, "seconds": 0.30335258100001283, "nonpadding_tokens_per_second": 144758.94635621426, "ranks": [{"compute_seconds": 0.29934302100002697, "nonpadding_tokens": 7829, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30034007899996595, "nonpadding_tokens": 3683, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3002301859999079, "nonpadding_tokens": 4008, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30016823200003273, "nonpadding_tokens": 4993, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3007690019999245, "nonpadding_tokens": 3052, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29846156699989024, "nonpadding_tokens": 4480, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3005994229999942, "nonpadding_tokens": 2622, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2960209320000331, "nonpadding_tokens": 13246, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037557299992840854, "prefetch": {"data_wait_seconds": 0.008262668001023599, "consumed_batches": 873, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1423, "tokens": 112217994, "loss_tokens": 67206, "loss": 2.4947939916078923, "lr": 0.00029977162230356344, "optimizer_lrs": [0.019984774820237566, 0.00029977162230356344], "gradient_norm": 0.41274505853652954, "seconds": 0.27096116699999584, "nonpadding_tokens_per_second": 248028.161171896, "ranks": [{"compute_seconds": 0.26856016600004295, "nonpadding_tokens": 4868, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2671028679999381, "nonpadding_tokens": 12930, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2679453250000279, "nonpadding_tokens": 6197, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26486234400010744, "nonpadding_tokens": 13790, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2686449160000848, "nonpadding_tokens": 2839, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2659219590000248, "nonpadding_tokens": 9268, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2682412909999812, "nonpadding_tokens": 7348, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2637963109999646, "nonpadding_tokens": 9966, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040898199995353934, "prefetch": {"data_wait_seconds": 0.008271032001019307, "consumed_batches": 874, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1424, "tokens": 112306346, "loss_tokens": 88352, "loss": 2.8135319163686163, "lr": 0.0002997711846105141, "optimizer_lrs": [0.019984745640700942, 0.0002997711846105141], "gradient_norm": 0.3788853585720062, "seconds": 0.26364650799996525, "nonpadding_tokens_per_second": 335115.3810844771, "ranks": [{"compute_seconds": 0.2610194579999643, "nonpadding_tokens": 6971, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2576566799999682, "nonpadding_tokens": 17341, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25786300400000073, "nonpadding_tokens": 6129, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25855471099998795, "nonpadding_tokens": 9990, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.25694331499994405, "nonpadding_tokens": 25146, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2580872009999666, "nonpadding_tokens": 13562, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2566868540000087, "nonpadding_tokens": 2297, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26139531299998, "nonpadding_tokens": 6916, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003608339999345844, "prefetch": {"data_wait_seconds": 0.00828085600103634, "consumed_batches": 875, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1425, "tokens": 112361140, "loss_tokens": 54794, "loss": 2.516829796601818, "lr": 0.0002997709129524096, "optimizer_lrs": [0.019984727530160642, 0.0002997709129524096], "gradient_norm": 1.1579691171646118, "seconds": 0.2755384069999991, "nonpadding_tokens_per_second": 198861.5692330695, "ranks": [{"compute_seconds": 0.2720548659999622, "nonpadding_tokens": 2924, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27012516100000994, "nonpadding_tokens": 3385, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2700476570000774, "nonpadding_tokens": 9832, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2725613310000199, "nonpadding_tokens": 12299, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27324360600005093, "nonpadding_tokens": 4044, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27173159400001623, "nonpadding_tokens": 9896, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26868656499993904, "nonpadding_tokens": 2982, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26992575099995975, "nonpadding_tokens": 9432, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035031599998092133, "prefetch": {"data_wait_seconds": 0.008289456001037365, "consumed_batches": 876, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1426, "tokens": 112446858, "loss_tokens": 85718, "loss": 2.1663095120044797, "lr": 0.0002997704876560732, "optimizer_lrs": [0.01998469917707155, 0.0002997704876560732], "gradient_norm": 0.5016571879386902, "seconds": 0.2518783740000572, "nonpadding_tokens_per_second": 340315.0442760145, "ranks": [{"compute_seconds": 0.2490048879999449, "nonpadding_tokens": 3122, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24895966700000827, "nonpadding_tokens": 3613, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24613318599995182, "nonpadding_tokens": 18898, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24664676600002622, "nonpadding_tokens": 17988, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2453451489999452, "nonpadding_tokens": 3595, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24659340699997756, "nonpadding_tokens": 11620, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.24874239000007492, "nonpadding_tokens": 7814, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2495512760000338, "nonpadding_tokens": 19068, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003826509999953487, "prefetch": {"data_wait_seconds": 0.008297675001088578, "consumed_batches": 877, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1427, "tokens": 112513602, "loss_tokens": 66744, "loss": 2.970316386117104, "lr": 0.000299770156227787, "optimizer_lrs": [0.01998467708185247, 0.000299770156227787], "gradient_norm": 0.5398191809654236, "seconds": 0.2977248079999981, "nonpadding_tokens_per_second": 224180.1764802899, "ranks": [{"compute_seconds": 0.2922091969999201, "nonpadding_tokens": 2585, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2949577470000122, "nonpadding_tokens": 13809, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29397763899999063, "nonpadding_tokens": 8016, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29504772200004936, "nonpadding_tokens": 4001, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29045346099997005, "nonpadding_tokens": 1501, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2940804479999315, "nonpadding_tokens": 18400, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2908350129999917, "nonpadding_tokens": 16009, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29515065399994, "nonpadding_tokens": 2423, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000623707000045215, "prefetch": {"data_wait_seconds": 0.008306069000994398, "consumed_batches": 878, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1428, "tokens": 112574677, "loss_tokens": 61075, "loss": 2.6705937883749487, "lr": 0.0002997698527405052, "optimizer_lrs": [0.019984656849367014, 0.0002997698527405052], "gradient_norm": 0.4798315167427063, "seconds": 0.33749824500000614, "nonpadding_tokens_per_second": 180963.90397526036, "ranks": [{"compute_seconds": 0.335156434000055, "nonpadding_tokens": 25062, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3338540270000294, "nonpadding_tokens": 9287, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3331244199999901, "nonpadding_tokens": 3255, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33398545000000013, "nonpadding_tokens": 6958, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.32995846400001483, "nonpadding_tokens": 2381, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33135999399996763, "nonpadding_tokens": 3321, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.32983619800006636, "nonpadding_tokens": 2446, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33441732799997226, "nonpadding_tokens": 8365, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003448780000780971, "prefetch": {"data_wait_seconds": 0.0083141760009795, "consumed_batches": 879, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1429, "tokens": 112631631, "loss_tokens": 56954, "loss": 2.519407383590266, "lr": 0.0002997695695505834, "optimizer_lrs": [0.019984637970038896, 0.0002997695695505834], "gradient_norm": 0.7605540156364441, "seconds": 0.3382021820000318, "nonpadding_tokens_per_second": 168402.22515180177, "ranks": [{"compute_seconds": 0.3337251559999004, "nonpadding_tokens": 2146, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.335633678000022, "nonpadding_tokens": 7409, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33256171300001824, "nonpadding_tokens": 3985, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3351844279999341, "nonpadding_tokens": 5184, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.33126040600006945, "nonpadding_tokens": 19295, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33327455699998154, "nonpadding_tokens": 4784, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.33551414800001567, "nonpadding_tokens": 3117, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3359161790000371, "nonpadding_tokens": 11034, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003499539999438639, "prefetch": {"data_wait_seconds": 0.008322906001083652, "consumed_batches": 880, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1430, "tokens": 112678642, "loss_tokens": 47011, "loss": 2.430609159292506, "lr": 0.0002997693356687465, "optimizer_lrs": [0.019984622377916435, 0.0002997693356687465], "gradient_norm": 0.4746589660644531, "seconds": 0.24489577600002121, "nonpadding_tokens_per_second": 191963.29462210048, "ranks": [{"compute_seconds": 0.24267469000005804, "nonpadding_tokens": 1503, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2412577569999712, "nonpadding_tokens": 4299, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23861546099999487, "nonpadding_tokens": 3066, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24075396299997465, "nonpadding_tokens": 14741, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23729692599999908, "nonpadding_tokens": 13105, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2402548539999998, "nonpadding_tokens": 5885, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2376706129999775, "nonpadding_tokens": 2370, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24145020000003115, "nonpadding_tokens": 2042, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035421300003690703, "prefetch": {"data_wait_seconds": 0.008330493001110995, "consumed_batches": 881, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1431, "tokens": 112752065, "loss_tokens": 73423, "loss": 2.7116402983397574, "lr": 0.00029976897014889493, "optimizer_lrs": [0.01998459800992633, 0.00029976897014889493], "gradient_norm": 0.5069602727890015, "seconds": 0.27514952199999243, "nonpadding_tokens_per_second": 266847.63784544036, "ranks": [{"compute_seconds": 0.27227848900008667, "nonpadding_tokens": 4293, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27246197100009795, "nonpadding_tokens": 14028, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2696933419999823, "nonpadding_tokens": 14746, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27223198900003354, "nonpadding_tokens": 15515, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2726694250000037, "nonpadding_tokens": 2659, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2710464309999452, "nonpadding_tokens": 5726, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2722743739999487, "nonpadding_tokens": 13629, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2691771649999737, "nonpadding_tokens": 2827, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037964700004522456, "prefetch": {"data_wait_seconds": 0.00834002700112535, "consumed_batches": 882, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1432, "tokens": 112799497, "loss_tokens": 47432, "loss": 2.6377409370256366, "lr": 0.0002997687338656545, "optimizer_lrs": [0.019984582257710302, 0.0002997687338656545], "gradient_norm": 0.4287235736846924, "seconds": 0.2708618789999946, "nonpadding_tokens_per_second": 175115.08143972134, "ranks": [{"compute_seconds": 0.26833945399994263, "nonpadding_tokens": 8755, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26861446700002034, "nonpadding_tokens": 9205, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2655865689999928, "nonpadding_tokens": 4292, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.266120403000059, "nonpadding_tokens": 4152, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26842282999996314, "nonpadding_tokens": 2060, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2657772259999547, "nonpadding_tokens": 3298, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.26450060599995595, "nonpadding_tokens": 10347, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2650407879999648, "nonpadding_tokens": 5323, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039800000001832814, "prefetch": {"data_wait_seconds": 0.008348587001094643, "consumed_batches": 883, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1433, "tokens": 112896098, "loss_tokens": 96601, "loss": 2.409252039316363, "lr": 0.0002997682522731656, "optimizer_lrs": [0.019984550151544375, 0.0002997682522731656], "gradient_norm": 0.5743048191070557, "seconds": 0.2740896820000671, "nonpadding_tokens_per_second": 352443.03723909, "ranks": [{"compute_seconds": 0.271052237000049, "nonpadding_tokens": 11064, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26834224399999584, "nonpadding_tokens": 11343, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2689789659999633, "nonpadding_tokens": 10459, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.271503310000071, "nonpadding_tokens": 11605, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27184605399997963, "nonpadding_tokens": 17105, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2710670520000349, "nonpadding_tokens": 19607, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.2717829900000197, "nonpadding_tokens": 5080, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26854482400005963, "nonpadding_tokens": 10338, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003695809999726407, "prefetch": {"data_wait_seconds": 0.008357260001005784, "consumed_batches": 884, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1434, "tokens": 112971606, "loss_tokens": 75508, "loss": 2.3886537684748634, "lr": 0.00029976787548875836, "optimizer_lrs": [0.019984525032583893, 0.00029976787548875836], "gradient_norm": 0.46973904967308044, "seconds": 0.2200297250000176, "nonpadding_tokens_per_second": 343171.81462638266, "ranks": [{"compute_seconds": 0.2157460789999277, "nonpadding_tokens": 9943, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21342360500000268, "nonpadding_tokens": 15423, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21660669899995355, "nonpadding_tokens": 7771, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21539590299994416, "nonpadding_tokens": 5631, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21764753899992684, "nonpadding_tokens": 11327, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21588987999996334, "nonpadding_tokens": 6218, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.21716727799991986, "nonpadding_tokens": 5137, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21776655100006792, "nonpadding_tokens": 14058, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036064900007204415, "prefetch": {"data_wait_seconds": 0.008366627000967242, "consumed_batches": 885, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1435, "tokens": 113033676, "loss_tokens": 62070, "loss": 2.4479514056710165, "lr": 0.00029976756553088367, "optimizer_lrs": [0.019984504368725578, 0.00029976756553088367], "gradient_norm": 0.3900173306465149, "seconds": 0.32324144999995497, "nonpadding_tokens_per_second": 192023.64053251417, "ranks": [{"compute_seconds": 0.3202870970000049, "nonpadding_tokens": 9966, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32074370300006194, "nonpadding_tokens": 3008, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31767461899994487, "nonpadding_tokens": 4003, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3177880539999478, "nonpadding_tokens": 13252, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3209105760000739, "nonpadding_tokens": 7281, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31944895900005577, "nonpadding_tokens": 13850, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19061014528}, {"compute_seconds": 0.3204706179999448, "nonpadding_tokens": 2314, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31704567200006295, "nonpadding_tokens": 8396, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003404159999718104, "prefetch": {"data_wait_seconds": 0.00837436100091793, "consumed_batches": 886, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1436, "tokens": 113097051, "loss_tokens": 63375, "loss": 2.4267702169625247, "lr": 0.0002997672488430628, "optimizer_lrs": [0.019984483256204187, 0.0002997672488430628], "gradient_norm": 0.44462335109710693, "seconds": 0.29318531400008396, "nonpadding_tokens_per_second": 216160.21326355333, "ranks": [{"compute_seconds": 0.29023649799989926, "nonpadding_tokens": 5925, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29088763599997947, "nonpadding_tokens": 4328, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2895773879999979, "nonpadding_tokens": 1693, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2881104069999765, "nonpadding_tokens": 12093, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29046604699999534, "nonpadding_tokens": 7205, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2890476300000273, "nonpadding_tokens": 10906, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2904866989999846, "nonpadding_tokens": 3395, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28692480800009434, "nonpadding_tokens": 17830, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041073600004892796, "prefetch": {"data_wait_seconds": 0.008384348000916475, "consumed_batches": 887, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1437, "tokens": 113163742, "loss_tokens": 66691, "loss": 2.7783185699719604, "lr": 0.00029976691535242686, "optimizer_lrs": [0.019984461023495124, 0.00029976691535242686], "gradient_norm": 0.3764845132827759, "seconds": 0.34273797399998784, "nonpadding_tokens_per_second": 194583.04903209343, "ranks": [{"compute_seconds": 0.3402693589999899, "nonpadding_tokens": 11814, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3377266339999778, "nonpadding_tokens": 2233, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33396613900004013, "nonpadding_tokens": 10195, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3345518070000253, "nonpadding_tokens": 5981, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3330357710000271, "nonpadding_tokens": 4907, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33706883599995763, "nonpadding_tokens": 8067, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33917612699997335, "nonpadding_tokens": 9630, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.339821465, "nonpadding_tokens": 13864, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035633599998163845, "prefetch": {"data_wait_seconds": 0.008391827000949093, "consumed_batches": 888, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1438, "tokens": 113196908, "loss_tokens": 33166, "loss": 2.4703268218959176, "lr": 0.00029976674941589395, "optimizer_lrs": [0.019984449961059598, 0.00029976674941589395], "gradient_norm": 0.427890419960022, "seconds": 0.22553408899989336, "nonpadding_tokens_per_second": 147055.3748529593, "ranks": [{"compute_seconds": 0.2228709760000811, "nonpadding_tokens": 4332, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2196113500000365, "nonpadding_tokens": 6037, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22199174900003982, "nonpadding_tokens": 5177, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22241411800007427, "nonpadding_tokens": 3106, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21886489100006656, "nonpadding_tokens": 1853, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2219956239999874, "nonpadding_tokens": 3786, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22282020999989527, "nonpadding_tokens": 6872, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22017478800000845, "nonpadding_tokens": 2003, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038827699995636067, "prefetch": {"data_wait_seconds": 0.008402148000939746, "consumed_batches": 889, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1439, "tokens": 113263533, "loss_tokens": 66625, "loss": 2.409391181988743, "lr": 0.00029976641589859896, "optimizer_lrs": [0.019984427726573266, 0.00029976641589859896], "gradient_norm": 0.5038633942604065, "seconds": 0.359137636000014, "nonpadding_tokens_per_second": 185513.83459013858, "ranks": [{"compute_seconds": 0.3545434939999268, "nonpadding_tokens": 18479, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35493943400001626, "nonpadding_tokens": 9247, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3551279160000149, "nonpadding_tokens": 7107, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3533550170000126, "nonpadding_tokens": 8456, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3515177210000502, "nonpadding_tokens": 9771, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3530008479999651, "nonpadding_tokens": 1972, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3553528649999862, "nonpadding_tokens": 1926, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35648365899999135, "nonpadding_tokens": 9667, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045062999993206176, "prefetch": {"data_wait_seconds": 0.008411913001012294, "consumed_batches": 890, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1440, "tokens": 113335046, "loss_tokens": 71513, "loss": 2.932304266357166, "lr": 0.00029976605764761734, "optimizer_lrs": [0.01998440384317449, 0.00029976605764761734], "gradient_norm": 0.4485793113708496, "seconds": 0.27747650599997087, "nonpadding_tokens_per_second": 257726.32440458765, "ranks": [{"compute_seconds": 0.27163210800006254, "nonpadding_tokens": 11193, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2726200209999661, "nonpadding_tokens": 9457, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27378094599998803, "nonpadding_tokens": 2125, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2701025530000152, "nonpadding_tokens": 16494, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27455102700002953, "nonpadding_tokens": 6438, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27176263299998027, "nonpadding_tokens": 10820, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26843188300006204, "nonpadding_tokens": 9977, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27497644000004584, "nonpadding_tokens": 5009, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003611149999187546, "prefetch": {"data_wait_seconds": 0.008419933001007962, "consumed_batches": 891, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1441, "tokens": 113401684, "loss_tokens": 66638, "loss": 2.4304143750562743, "lr": 0.00029976572357152915, "optimizer_lrs": [0.019984381571435277, 0.00029976572357152915], "gradient_norm": 0.7243877649307251, "seconds": 0.32085487300003024, "nonpadding_tokens_per_second": 207688.913610465, "ranks": [{"compute_seconds": 0.312978461000057, "nonpadding_tokens": 4478, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31575006299999586, "nonpadding_tokens": 10523, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3163011550000192, "nonpadding_tokens": 4494, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31843427299997984, "nonpadding_tokens": 6739, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31764611900007367, "nonpadding_tokens": 9344, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3143325480000385, "nonpadding_tokens": 19611, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3169902430000775, "nonpadding_tokens": 4002, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3176689470000156, "nonpadding_tokens": 7447, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003357429999368833, "prefetch": {"data_wait_seconds": 0.008428593001056583, "consumed_batches": 892, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1442, "tokens": 113455624, "loss_tokens": 53940, "loss": 2.4736466444197256, "lr": 0.0002997654529798775, "optimizer_lrs": [0.019984363531991838, 0.0002997654529798775], "gradient_norm": 0.5834755897521973, "seconds": 0.3869406410000238, "nonpadding_tokens_per_second": 139401.22665997414, "ranks": [{"compute_seconds": 0.3837832100000469, "nonpadding_tokens": 2167, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3801428560000204, "nonpadding_tokens": 5232, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3804954870000756, "nonpadding_tokens": 6105, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3808659970000008, "nonpadding_tokens": 11061, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3791030840000076, "nonpadding_tokens": 6241, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3813023499999417, "nonpadding_tokens": 3063, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37968693400000575, "nonpadding_tokens": 14869, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3844965629999706, "nonpadding_tokens": 5202, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003360309999607125, "prefetch": {"data_wait_seconds": 0.008436894001150108, "consumed_batches": 893, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1443, "tokens": 113526881, "loss_tokens": 71257, "loss": 2.6886075666250333, "lr": 0.0002997650952777409, "optimizer_lrs": [0.019984339685182727, 0.0002997650952777409], "gradient_norm": 0.3971366286277771, "seconds": 0.3414103930000465, "nonpadding_tokens_per_second": 208713.62284507343, "ranks": [{"compute_seconds": 0.337849641000048, "nonpadding_tokens": 13174, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.339042371000005, "nonpadding_tokens": 8043, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33701950299996497, "nonpadding_tokens": 3429, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3380353280000463, "nonpadding_tokens": 18140, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3380153260000043, "nonpadding_tokens": 15909, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3353038950000382, "nonpadding_tokens": 6016, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33488664599997264, "nonpadding_tokens": 3596, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3384405780000179, "nonpadding_tokens": 2950, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003454530000226441, "prefetch": {"data_wait_seconds": 0.008445335001127319, "consumed_batches": 894, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1444, "tokens": 113576160, "loss_tokens": 49279, "loss": 2.423059898739828, "lr": 0.0002997648477434136, "optimizer_lrs": [0.019984323182894243, 0.0002997648477434136], "gradient_norm": 0.42013898491859436, "seconds": 0.28713259399989965, "nonpadding_tokens_per_second": 171624.5422141703, "ranks": [{"compute_seconds": 0.28475812499993935, "nonpadding_tokens": 9389, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28086898300000485, "nonpadding_tokens": 10243, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28142195899999933, "nonpadding_tokens": 4346, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28395500900001025, "nonpadding_tokens": 3946, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28029505699998936, "nonpadding_tokens": 6161, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28311896799993974, "nonpadding_tokens": 2063, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2842563209999298, "nonpadding_tokens": 8748, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28137344599997505, "nonpadding_tokens": 4383, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039054800004123535, "prefetch": {"data_wait_seconds": 0.008453408001130924, "consumed_batches": 895, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1445, "tokens": 113650125, "loss_tokens": 73965, "loss": 2.4265923916717367, "lr": 0.0002997644759639034, "optimizer_lrs": [0.01998429839759356, 0.0002997644759639034], "gradient_norm": 0.414870947599411, "seconds": 0.28603857300004165, "nonpadding_tokens_per_second": 258584.0057312453, "ranks": [{"compute_seconds": 0.2836797040000647, "nonpadding_tokens": 13922, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28309872399995584, "nonpadding_tokens": 4986, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2822091979999186, "nonpadding_tokens": 3665, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2809920519999878, "nonpadding_tokens": 18580, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2832215799999176, "nonpadding_tokens": 5827, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2816962299999659, "nonpadding_tokens": 10998, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28299664600001506, "nonpadding_tokens": 6005, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27982020600006763, "nonpadding_tokens": 9982, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036126099996636185, "prefetch": {"data_wait_seconds": 0.008463375001113604, "consumed_batches": 896, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1446, "tokens": 113712025, "loss_tokens": 61900, "loss": 3.207164277059774, "lr": 0.00029976416460268847, "optimizer_lrs": [0.01998427764017923, 0.00029976416460268847], "gradient_norm": 0.5677536129951477, "seconds": 0.271559907999972, "nonpadding_tokens_per_second": 227942.3367605736, "ranks": [{"compute_seconds": 0.2686100130000568, "nonpadding_tokens": 11630, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2657661470000221, "nonpadding_tokens": 5077, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26833452999994734, "nonpadding_tokens": 2732, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2687380169999187, "nonpadding_tokens": 17134, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.26920154900005855, "nonpadding_tokens": 12641, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26794211900005394, "nonpadding_tokens": 2540, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26914131000000907, "nonpadding_tokens": 6343, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26538706300004833, "nonpadding_tokens": 3803, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037442199993620306, "prefetch": {"data_wait_seconds": 0.008471753001074944, "consumed_batches": 897, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1447, "tokens": 113776544, "loss_tokens": 64519, "loss": 2.6200566887273515, "lr": 0.00029976383984900684, "optimizer_lrs": [0.01998425598993379, 0.00029976383984900684], "gradient_norm": 0.4329869747161865, "seconds": 0.3020206500000313, "nonpadding_tokens_per_second": 213624.46574429038, "ranks": [{"compute_seconds": 0.2983363470000313, "nonpadding_tokens": 3295, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29896838799993475, "nonpadding_tokens": 2144, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29608228600000075, "nonpadding_tokens": 4660, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2963523900000382, "nonpadding_tokens": 6709, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29879082299999027, "nonpadding_tokens": 6877, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2976022049999756, "nonpadding_tokens": 10219, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29871722699999737, "nonpadding_tokens": 12570, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2996249220000209, "nonpadding_tokens": 18045, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040071299997634924, "prefetch": {"data_wait_seconds": 0.008482647001031296, "consumed_batches": 898, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1448, "tokens": 113832344, "loss_tokens": 55800, "loss": 2.6271040546594984, "lr": 0.00029976355880201284, "optimizer_lrs": [0.019984237253467523, 0.00029976355880201284], "gradient_norm": 0.3667142391204834, "seconds": 0.26473572200006856, "nonpadding_tokens_per_second": 210776.23970967374, "ranks": [{"compute_seconds": 0.2598588320000772, "nonpadding_tokens": 6934, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26245437199997923, "nonpadding_tokens": 8633, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2612522939999735, "nonpadding_tokens": 15693, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26225307199990766, "nonpadding_tokens": 2105, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2583241239999552, "nonpadding_tokens": 4618, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26144530600004146, "nonpadding_tokens": 8653, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2582330989999946, "nonpadding_tokens": 2703, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25909981699999207, "nonpadding_tokens": 6461, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037219599994386954, "prefetch": {"data_wait_seconds": 0.008491414001085218, "consumed_batches": 899, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1449, "tokens": 113899286, "loss_tokens": 66942, "loss": 2.5088742866959457, "lr": 0.0002997632214159494, "optimizer_lrs": [0.019984214761063296, 0.0002997632214159494], "gradient_norm": 0.3813101351261139, "seconds": 0.2792252340000232, "nonpadding_tokens_per_second": 239741.94252083407, "ranks": [{"compute_seconds": 0.27414440100005777, "nonpadding_tokens": 3891, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27626255699999547, "nonpadding_tokens": 4686, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27317697900002713, "nonpadding_tokens": 12995, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27346090700007153, "nonpadding_tokens": 13490, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2761721529999477, "nonpadding_tokens": 3996, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2734601260000318, "nonpadding_tokens": 17011, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27242006200003743, "nonpadding_tokens": 7261, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2769198259999257, "nonpadding_tokens": 3612, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034019199995327654, "prefetch": {"data_wait_seconds": 0.008499878001046568, "consumed_batches": 900, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1450, "tokens": 113949702, "loss_tokens": 50416, "loss": 2.5772473024436686, "lr": 0.00029976296716186835, "optimizer_lrs": [0.019984197810791225, 0.00029976296716186835], "gradient_norm": 0.45286184549331665, "seconds": 0.2966400650000196, "nonpadding_tokens_per_second": 169956.81281285002, "ranks": [{"compute_seconds": 0.29412517699995533, "nonpadding_tokens": 6128, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2888798039999756, "nonpadding_tokens": 2834, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29198005699993246, "nonpadding_tokens": 5618, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2918973250000363, "nonpadding_tokens": 4269, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28719908400000804, "nonpadding_tokens": 7582, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29027449600005184, "nonpadding_tokens": 10553, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29250679999995555, "nonpadding_tokens": 6689, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28780410799993206, "nonpadding_tokens": 6743, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037117900001248927, "prefetch": {"data_wait_seconds": 0.008508300000997338, "consumed_batches": 901, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1451, "tokens": 114006469, "loss_tokens": 56767, "loss": 2.555253547395494, "lr": 0.000299762680715758, "optimizer_lrs": [0.019984178714383866, 0.000299762680715758], "gradient_norm": 0.4814767837524414, "seconds": 0.2838398320000124, "nonpadding_tokens_per_second": 199996.59526291405, "ranks": [{"compute_seconds": 0.27931164500000705, "nonpadding_tokens": 3956, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27894056500008446, "nonpadding_tokens": 5734, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2794139280000536, "nonpadding_tokens": 11488, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27630893499997455, "nonpadding_tokens": 4107, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27470005000009223, "nonpadding_tokens": 6557, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2775178180000921, "nonpadding_tokens": 5546, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27500910000003387, "nonpadding_tokens": 13902, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28134886500004086, "nonpadding_tokens": 5477, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005067700000154218, "prefetch": {"data_wait_seconds": 0.008521494001072938, "consumed_batches": 902, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1452, "tokens": 114066284, "loss_tokens": 59815, "loss": 2.748353255872273, "lr": 0.0002997623787024571, "optimizer_lrs": [0.019984158580163808, 0.0002997623787024571], "gradient_norm": 0.4121739864349365, "seconds": 0.3440133669998886, "nonpadding_tokens_per_second": 173874.05763224128, "ranks": [{"compute_seconds": 0.31181248400002914, "nonpadding_tokens": 5375, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3181665619999876, "nonpadding_tokens": 5700, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3149259719999691, "nonpadding_tokens": 3816, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31806921199995486, "nonpadding_tokens": 4316, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.31879417499999363, "nonpadding_tokens": 3280, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3172847069999989, "nonpadding_tokens": 7528, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31389476399999694, "nonpadding_tokens": 19739, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31872335600007773, "nonpadding_tokens": 10061, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.013800422000031176, "prefetch": {"data_wait_seconds": 0.008529871001044853, "consumed_batches": 903, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1453, "tokens": 114129840, "loss_tokens": 63556, "loss": 2.9601933236043805, "lr": 0.0002997620575901301, "optimizer_lrs": [0.019984137172675342, 0.0002997620575901301], "gradient_norm": 0.3777126967906952, "seconds": 0.30790130599996246, "nonpadding_tokens_per_second": 206416.79252899223, "ranks": [{"compute_seconds": 0.30306309699994927, "nonpadding_tokens": 7543, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.305226274000006, "nonpadding_tokens": 6661, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3044310420000329, "nonpadding_tokens": 2567, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30507664600008866, "nonpadding_tokens": 15731, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3055387050000036, "nonpadding_tokens": 12291, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30432624399998076, "nonpadding_tokens": 10290, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30486776300006113, "nonpadding_tokens": 4087, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3017433390000406, "nonpadding_tokens": 4386, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003918419999990874, "prefetch": {"data_wait_seconds": 0.008538822001085578, "consumed_batches": 904, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1454, "tokens": 114225573, "loss_tokens": 95733, "loss": 2.7522513788348846, "lr": 0.00029976157349681114, "optimizer_lrs": [0.01998410489978741, 0.00029976157349681114], "gradient_norm": 0.42139169573783875, "seconds": 0.2907521120000638, "nonpadding_tokens_per_second": 329259.86105985363, "ranks": [{"compute_seconds": 0.2877822649999189, "nonpadding_tokens": 1447, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2833055899999408, "nonpadding_tokens": 11572, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28667175500004305, "nonpadding_tokens": 20732, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2812780340000245, "nonpadding_tokens": 22454, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28783199200006493, "nonpadding_tokens": 9773, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28185999599998013, "nonpadding_tokens": 4984, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27929526399998394, "nonpadding_tokens": 17556, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2882264509999004, "nonpadding_tokens": 7215, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003672130000040852, "prefetch": {"data_wait_seconds": 0.008546874001083893, "consumed_batches": 905, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1455, "tokens": 114286411, "loss_tokens": 60838, "loss": 2.7129749293204903, "lr": 0.0002997612656017076, "optimizer_lrs": [0.019984084373447175, 0.0002997612656017076], "gradient_norm": 0.4417698383331299, "seconds": 0.25101151600006233, "nonpadding_tokens_per_second": 242371.3500060487, "ranks": [{"compute_seconds": 0.2484429830000181, "nonpadding_tokens": 5441, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2476919689999022, "nonpadding_tokens": 12101, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.245457311999985, "nonpadding_tokens": 8425, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24608488399996986, "nonpadding_tokens": 4628, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2441782639999701, "nonpadding_tokens": 5191, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24864709999997103, "nonpadding_tokens": 5973, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24393842100005259, "nonpadding_tokens": 5126, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24628350299997237, "nonpadding_tokens": 13953, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042365599995264347, "prefetch": {"data_wait_seconds": 0.008555961001164292, "consumed_batches": 906, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1456, "tokens": 114366877, "loss_tokens": 80466, "loss": 2.849901472360997, "lr": 0.0002997608580663466, "optimizer_lrs": [0.019984057204423106, 0.0002997608580663466], "gradient_norm": 2.981978178024292, "seconds": 0.23949167199998556, "nonpadding_tokens_per_second": 335986.6308837864, "ranks": [{"compute_seconds": 0.23555831900000612, "nonpadding_tokens": 6644, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23315076100004717, "nonpadding_tokens": 7176, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.235843333000048, "nonpadding_tokens": 4522, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2323497419999967, "nonpadding_tokens": 9853, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23052660400003333, "nonpadding_tokens": 10582, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23317420499995478, "nonpadding_tokens": 12469, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2303208910000194, "nonpadding_tokens": 13206, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2372266380000383, "nonpadding_tokens": 16014, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034877100006269757, "prefetch": {"data_wait_seconds": 0.008564854001136268, "consumed_batches": 907, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1457, "tokens": 114461163, "loss_tokens": 94286, "loss": 2.8028448682731266, "lr": 0.0002997603800951485, "optimizer_lrs": [0.019984025339676567, 0.0002997603800951485], "gradient_norm": 1.6621426343917847, "seconds": 0.37025249700002405, "nonpadding_tokens_per_second": 254653.24545804178, "ranks": [{"compute_seconds": 0.3642298899999332, "nonpadding_tokens": 8131, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36549117300000944, "nonpadding_tokens": 19370, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3625813039999457, "nonpadding_tokens": 24945, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3673369510000839, "nonpadding_tokens": 16719, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3678330840000399, "nonpadding_tokens": 8418, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3645774329999085, "nonpadding_tokens": 4550, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36163681999994424, "nonpadding_tokens": 10673, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36220751199994083, "nonpadding_tokens": 1480, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003833580000218717, "prefetch": {"data_wait_seconds": 0.00857309800115047, "consumed_batches": 908, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1458, "tokens": 114544112, "loss_tokens": 82949, "loss": 3.008655732739394, "lr": 0.00029975995920125177, "optimizer_lrs": [0.019983997280083453, 0.00029975995920125177], "gradient_norm": 1.8916759490966797, "seconds": 0.276906441000051, "nonpadding_tokens_per_second": 299556.0511356423, "ranks": [{"compute_seconds": 0.2735521910000216, "nonpadding_tokens": 6850, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2720116880000205, "nonpadding_tokens": 9850, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2735752419999926, "nonpadding_tokens": 10509, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27378272199996445, "nonpadding_tokens": 13551, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2744406709999794, "nonpadding_tokens": 4377, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.270942161999983, "nonpadding_tokens": 12280, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2680353659999355, "nonpadding_tokens": 12277, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2691219280000041, "nonpadding_tokens": 13255, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039028900005178, "prefetch": {"data_wait_seconds": 0.00858139300119376, "consumed_batches": 909, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1459, "tokens": 114609391, "loss_tokens": 65279, "loss": 2.589619173087823, "lr": 0.000299759627707745, "optimizer_lrs": [0.019983975180516336, 0.000299759627707745], "gradient_norm": 0.41721656918525696, "seconds": 0.3393269230000442, "nonpadding_tokens_per_second": 192377.89746495153, "ranks": [{"compute_seconds": 0.33700562800004263, "nonpadding_tokens": 11005, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3343431609999925, "nonpadding_tokens": 11850, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3357638139999608, "nonpadding_tokens": 15082, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3363273060000438, "nonpadding_tokens": 4801, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3311273200000642, "nonpadding_tokens": 8525, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3338667570000098, "nonpadding_tokens": 4286, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3315467529999978, "nonpadding_tokens": 6377, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3315903499999422, "nonpadding_tokens": 3353, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003685570000016014, "prefetch": {"data_wait_seconds": 0.008590917001200182, "consumed_batches": 910, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1460, "tokens": 114651072, "loss_tokens": 41681, "loss": 2.695598151735803, "lr": 0.0002997594159277757, "optimizer_lrs": [0.019983961061851716, 0.0002997594159277757], "gradient_norm": 0.4386732578277588, "seconds": 0.29391545000009955, "nonpadding_tokens_per_second": 141812.8921088901, "ranks": [{"compute_seconds": 0.2899448840000787, "nonpadding_tokens": 1761, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28786335100005545, "nonpadding_tokens": 4436, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.289796211999942, "nonpadding_tokens": 5535, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2855076209999652, "nonpadding_tokens": 3156, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2909149609999986, "nonpadding_tokens": 2690, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28660683499992956, "nonpadding_tokens": 5067, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2912956310000254, "nonpadding_tokens": 8002, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2910412760001009, "nonpadding_tokens": 11034, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004076409999242969, "prefetch": {"data_wait_seconds": 0.008598899001256655, "consumed_batches": 911, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1461, "tokens": 114722730, "loss_tokens": 71658, "loss": 2.3604532553936757, "lr": 0.00029975905161774835, "optimizer_lrs": [0.01998393677451656, 0.00029975905161774835], "gradient_norm": 0.5215812921524048, "seconds": 0.24356789499995557, "nonpadding_tokens_per_second": 294201.3355250004, "ranks": [{"compute_seconds": 0.24087785099993653, "nonpadding_tokens": 3371, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2374528159999727, "nonpadding_tokens": 6539, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2341588929999716, "nonpadding_tokens": 11061, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23448977800001103, "nonpadding_tokens": 25391, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23304771699997673, "nonpadding_tokens": 12350, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2362782850000258, "nonpadding_tokens": 3534, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.232962577999956, "nonpadding_tokens": 7930, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24091906700004984, "nonpadding_tokens": 1482, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040427099997941696, "prefetch": {"data_wait_seconds": 0.008626461001270513, "consumed_batches": 912, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1462, "tokens": 114788520, "loss_tokens": 65790, "loss": 2.7363353093175253, "lr": 0.0002997587168982197, "optimizer_lrs": [0.019983914459881312, 0.0002997587168982197], "gradient_norm": 0.40867939591407776, "seconds": 0.2947363509999832, "nonpadding_tokens_per_second": 223216.4433629829, "ranks": [{"compute_seconds": 0.29227447499999926, "nonpadding_tokens": 5465, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2901867680000123, "nonpadding_tokens": 11227, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29048019000003933, "nonpadding_tokens": 11981, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2914466329999641, "nonpadding_tokens": 7764, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2915256999999656, "nonpadding_tokens": 3226, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2890341500000204, "nonpadding_tokens": 15369, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2860493559999213, "nonpadding_tokens": 6626, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28654698500008635, "nonpadding_tokens": 4132, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043457600008878217, "prefetch": {"data_wait_seconds": 0.00863557000123194, "consumed_batches": 913, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1463, "tokens": 114856561, "loss_tokens": 68041, "loss": 2.411891359621405, "lr": 0.0002997583704821152, "optimizer_lrs": [0.01998389136547435, 0.0002997583704821152], "gradient_norm": 0.41869813203811646, "seconds": 0.28674609100005455, "nonpadding_tokens_per_second": 237286.58257450163, "ranks": [{"compute_seconds": 0.2824830079999856, "nonpadding_tokens": 3581, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.282607584999937, "nonpadding_tokens": 4341, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27960399699998106, "nonpadding_tokens": 8647, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28012454600002457, "nonpadding_tokens": 12981, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.28427991599994584, "nonpadding_tokens": 4785, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2820909549999442, "nonpadding_tokens": 18885, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2829003809999904, "nonpadding_tokens": 5076, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2838130339999907, "nonpadding_tokens": 9745, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038115800009563827, "prefetch": {"data_wait_seconds": 0.008644328001267354, "consumed_batches": 914, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1464, "tokens": 114944802, "loss_tokens": 88241, "loss": 2.784863151766186, "lr": 0.000299757920852277, "optimizer_lrs": [0.019983861390151802, 0.000299757920852277], "gradient_norm": 0.5221282839775085, "seconds": 0.281822582000018, "nonpadding_tokens_per_second": 313108.33707426034, "ranks": [{"compute_seconds": 0.27826024199998756, "nonpadding_tokens": 10439, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27817598100000396, "nonpadding_tokens": 5929, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2774205150000171, "nonpadding_tokens": 26568, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2783897289999686, "nonpadding_tokens": 9291, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2741554570000062, "nonpadding_tokens": 15879, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2771689480000532, "nonpadding_tokens": 10978, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2736368959999709, "nonpadding_tokens": 3356, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2793554149999409, "nonpadding_tokens": 5801, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003559289999657267, "prefetch": {"data_wait_seconds": 0.008653859001199748, "consumed_batches": 915, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1465, "tokens": 114992324, "loss_tokens": 47522, "loss": 2.6370204194899203, "lr": 0.0002997576785321003, "optimizer_lrs": [0.019983845235473353, 0.0002997576785321003], "gradient_norm": 0.45602357387542725, "seconds": 0.2540665890001037, "nonpadding_tokens_per_second": 187045.4520880768, "ranks": [{"compute_seconds": 0.2514271259999532, "nonpadding_tokens": 1706, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24973289599995496, "nonpadding_tokens": 3281, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25076367100007246, "nonpadding_tokens": 9236, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24734172199998739, "nonpadding_tokens": 4905, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24564026999996713, "nonpadding_tokens": 2047, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24880547700001898, "nonpadding_tokens": 1859, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2516967529999192, "nonpadding_tokens": 16464, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24686018800002785, "nonpadding_tokens": 8024, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004113269999379554, "prefetch": {"data_wait_seconds": 0.008662947001155885, "consumed_batches": 916, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1466, "tokens": 115053083, "loss_tokens": 60759, "loss": 2.4742697583897035, "lr": 0.0002997573685385173, "optimizer_lrs": [0.019983824569234487, 0.0002997573685385173], "gradient_norm": 0.3609143793582916, "seconds": 0.23919773000000077, "nonpadding_tokens_per_second": 254011.60788607737, "ranks": [{"compute_seconds": 0.23459670499994445, "nonpadding_tokens": 6808, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23124237700005779, "nonpadding_tokens": 7238, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23393105200000264, "nonpadding_tokens": 10529, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2310499529999106, "nonpadding_tokens": 3822, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2299194060000218, "nonpadding_tokens": 7756, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2319961800000101, "nonpadding_tokens": 11020, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2366724040000463, "nonpadding_tokens": 9544, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23511496399999032, "nonpadding_tokens": 4042, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038394299997435155, "prefetch": {"data_wait_seconds": 0.008673076001130653, "consumed_batches": 917, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1467, "tokens": 115109541, "loss_tokens": 56458, "loss": 2.6504423088844806, "lr": 0.00029975708031126724, "optimizer_lrs": [0.019983805354084484, 0.00029975708031126724], "gradient_norm": 0.4086330533027649, "seconds": 0.2134458729999551, "nonpadding_tokens_per_second": 264507.33952589415, "ranks": [{"compute_seconds": 0.20750517699991633, "nonpadding_tokens": 4273, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21018090300003678, "nonpadding_tokens": 13492, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21030813499999113, "nonpadding_tokens": 5413, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2078199490000543, "nonpadding_tokens": 11978, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2068210720000252, "nonpadding_tokens": 1807, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20908205399996405, "nonpadding_tokens": 6169, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21045555399996374, "nonpadding_tokens": 10449, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21088480799994613, "nonpadding_tokens": 2877, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003851690000828967, "prefetch": {"data_wait_seconds": 0.008682713001078923, "consumed_batches": 918, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1468, "tokens": 115154315, "loss_tokens": 44774, "loss": 2.7488009224103274, "lr": 0.0002997568516111964, "optimizer_lrs": [0.019983790107413097, 0.0002997568516111964], "gradient_norm": 0.45504897832870483, "seconds": 0.25279261999992286, "nonpadding_tokens_per_second": 177117.51237047056, "ranks": [{"compute_seconds": 0.24847997799997756, "nonpadding_tokens": 4123, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24848471400002836, "nonpadding_tokens": 2157, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2499830970000403, "nonpadding_tokens": 7334, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2504429559999153, "nonpadding_tokens": 2061, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24471924599993145, "nonpadding_tokens": 8513, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2485577720000265, "nonpadding_tokens": 12453, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24398478000000523, "nonpadding_tokens": 4641, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2447576270000127, "nonpadding_tokens": 3492, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003258640000467494, "prefetch": {"data_wait_seconds": 0.00869191800116198, "consumed_batches": 919, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1469, "tokens": 115209199, "loss_tokens": 54884, "loss": 2.73727173447635, "lr": 0.0002997565711238296, "optimizer_lrs": [0.019983771408255305, 0.0002997565711238296], "gradient_norm": 0.403361052274704, "seconds": 0.24924784099994213, "nonpadding_tokens_per_second": 220198.4971256491, "ranks": [{"compute_seconds": 0.24330310200002714, "nonpadding_tokens": 6045, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2455038319999403, "nonpadding_tokens": 9636, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24570651200008342, "nonpadding_tokens": 4150, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24633133899999393, "nonpadding_tokens": 8106, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.24693642100010038, "nonpadding_tokens": 7175, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24459015700006148, "nonpadding_tokens": 5325, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24656262200005585, "nonpadding_tokens": 7347, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24206556000001456, "nonpadding_tokens": 7100, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003381290000561421, "prefetch": {"data_wait_seconds": 0.008699525001205188, "consumed_batches": 920, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1470, "tokens": 115245995, "loss_tokens": 36796, "loss": 2.7032509053293836, "lr": 0.0002997563829856158, "optimizer_lrs": [0.01998375886570772, 0.0002997563829856158], "gradient_norm": 0.41612035036087036, "seconds": 0.3532032599999866, "nonpadding_tokens_per_second": 104177.9738952619, "ranks": [{"compute_seconds": 0.35005219800007126, "nonpadding_tokens": 2617, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3474487880000652, "nonpadding_tokens": 5490, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3499743679999483, "nonpadding_tokens": 3962, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.35051459399994656, "nonpadding_tokens": 3488, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3458826690000478, "nonpadding_tokens": 2506, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3482906610000782, "nonpadding_tokens": 10542, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34561127599999963, "nonpadding_tokens": 3828, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3467026440000609, "nonpadding_tokens": 4363, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036840099994606135, "prefetch": {"data_wait_seconds": 0.008707161001169084, "consumed_batches": 921, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1471, "tokens": 115272071, "loss_tokens": 26076, "loss": 2.5687542543910107, "lr": 0.0002997562496148793, "optimizer_lrs": [0.01998374997432529, 0.0002997562496148793], "gradient_norm": 0.5175032615661621, "seconds": 0.22372150800003965, "nonpadding_tokens_per_second": 116555.62414676455, "ranks": [{"compute_seconds": 0.2205819609999935, "nonpadding_tokens": 3636, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21890959500001372, "nonpadding_tokens": 4748, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21613386499996068, "nonpadding_tokens": 2671, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21615013699999963, "nonpadding_tokens": 2764, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.21469158800005061, "nonpadding_tokens": 2104, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.218262870999979, "nonpadding_tokens": 2722, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22056377599994903, "nonpadding_tokens": 3195, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2212978769999836, "nonpadding_tokens": 4236, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035470400007397984, "prefetch": {"data_wait_seconds": 0.008714788001228158, "consumed_batches": 922, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1472, "tokens": 115341168, "loss_tokens": 69097, "loss": 3.0374401204104373, "lr": 0.0002997558960285997, "optimizer_lrs": [0.019983726401906647, 0.0002997558960285997], "gradient_norm": 0.4242995083332062, "seconds": 0.3203456459999643, "nonpadding_tokens_per_second": 215695.14323914895, "ranks": [{"compute_seconds": 0.3180578610000566, "nonpadding_tokens": 8591, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3142836360000274, "nonpadding_tokens": 5512, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31450590200006445, "nonpadding_tokens": 1889, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3154626460000145, "nonpadding_tokens": 3703, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3177848059999633, "nonpadding_tokens": 10566, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31614763299990045, "nonpadding_tokens": 12299, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3173034970000117, "nonpadding_tokens": 18315, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3139849780000077, "nonpadding_tokens": 8222, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003451300000278934, "prefetch": {"data_wait_seconds": 0.008723175001208006, "consumed_batches": 923, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1473, "tokens": 115382163, "loss_tokens": 40995, "loss": 2.5373582144163924, "lr": 0.000299755686126123, "optimizer_lrs": [0.0199837124084082, 0.000299755686126123], "gradient_norm": 0.43383264541625977, "seconds": 0.2810188460000518, "nonpadding_tokens_per_second": 145879.8958984852, "ranks": [{"compute_seconds": 0.2604133739999952, "nonpadding_tokens": 8110, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2613895559999264, "nonpadding_tokens": 3445, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.264152107999962, "nonpadding_tokens": 7580, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2642991359999769, "nonpadding_tokens": 6515, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2601369159999649, "nonpadding_tokens": 5614, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26268293299995094, "nonpadding_tokens": 2625, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2639493939999511, "nonpadding_tokens": 2198, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2608944500000234, "nonpadding_tokens": 4908, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005733140000074854, "prefetch": {"data_wait_seconds": 0.00873132700128565, "consumed_batches": 924, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1474, "tokens": 115471665, "loss_tokens": 89502, "loss": 2.6620478802149674, "lr": 0.0002997552275450717, "optimizer_lrs": [0.019983681836338116, 0.0002997552275450717], "gradient_norm": 0.42911627888679504, "seconds": 0.2636225190000232, "nonpadding_tokens_per_second": 339508.1738066243, "ranks": [{"compute_seconds": 0.25652899199997137, "nonpadding_tokens": 13932, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25761980499999027, "nonpadding_tokens": 3933, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25632048199997826, "nonpadding_tokens": 7719, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.260419359000025, "nonpadding_tokens": 14671, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.261142212999971, "nonpadding_tokens": 8040, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25804709099998036, "nonpadding_tokens": 12163, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26065956400009327, "nonpadding_tokens": 15702, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25545431099999405, "nonpadding_tokens": 13342, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041105599996171804, "prefetch": {"data_wait_seconds": 0.00874035800127615, "consumed_batches": 925, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1475, "tokens": 115515077, "loss_tokens": 43412, "loss": 2.8390116068137843, "lr": 0.00029975500496045844, "optimizer_lrs": [0.019983666997363897, 0.00029975500496045844], "gradient_norm": 0.41642722487449646, "seconds": 0.2130363710000438, "nonpadding_tokens_per_second": 203777.41038402816, "ranks": [{"compute_seconds": 0.20737488099996426, "nonpadding_tokens": 4062, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20875617499996224, "nonpadding_tokens": 7729, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20902653400003146, "nonpadding_tokens": 3809, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2097311230000969, "nonpadding_tokens": 5891, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.20501365999996324, "nonpadding_tokens": 3175, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2070050730000048, "nonpadding_tokens": 2436, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20492226300007133, "nonpadding_tokens": 3499, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21057493000000704, "nonpadding_tokens": 12811, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003987960000131352, "prefetch": {"data_wait_seconds": 0.008749002001266604, "consumed_batches": 926, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1476, "tokens": 115585658, "loss_tokens": 70581, "loss": 2.5532911654694606, "lr": 0.00029975464285758124, "optimizer_lrs": [0.019983642857172083, 0.00029975464285758124], "gradient_norm": 0.6054818630218506, "seconds": 0.31787958100005653, "nonpadding_tokens_per_second": 222036.91026001272, "ranks": [{"compute_seconds": 0.3144698559999597, "nonpadding_tokens": 8280, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31438096500005486, "nonpadding_tokens": 12393, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31149142599997504, "nonpadding_tokens": 8623, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31214492499998414, "nonpadding_tokens": 7210, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3153170790000104, "nonpadding_tokens": 9734, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31282853000004707, "nonpadding_tokens": 4933, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31051748599998064, "nonpadding_tokens": 2071, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3112125699999524, "nonpadding_tokens": 17337, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005776319999313273, "prefetch": {"data_wait_seconds": 0.008754970001248807, "consumed_batches": 927, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1477, "tokens": 115674083, "loss_tokens": 88425, "loss": 2.332136344359627, "lr": 0.00029975418883233264, "optimizer_lrs": [0.019983612588822176, 0.00029975418883233264], "gradient_norm": 0.5290794968605042, "seconds": 0.30104740499996296, "nonpadding_tokens_per_second": 293724.5049496802, "ranks": [{"compute_seconds": 0.29573182499996165, "nonpadding_tokens": 10085, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2943206600000394, "nonpadding_tokens": 15167, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29110975300000064, "nonpadding_tokens": 4320, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29761764299996685, "nonpadding_tokens": 11690, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29805713100006415, "nonpadding_tokens": 18448, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2934625060000826, "nonpadding_tokens": 2106, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29720887200005563, "nonpadding_tokens": 9995, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29789554000001317, "nonpadding_tokens": 16614, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005668759999934991, "prefetch": {"data_wait_seconds": 0.008764533001297536, "consumed_batches": 928, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1478, "tokens": 115715468, "loss_tokens": 41385, "loss": 2.4704969342757037, "lr": 0.00029975397619368653, "optimizer_lrs": [0.019983598412912435, 0.00029975397619368653], "gradient_norm": 0.8212634325027466, "seconds": 0.2426958870000817, "nonpadding_tokens_per_second": 170522.0492673041, "ranks": [{"compute_seconds": 0.23883029700004954, "nonpadding_tokens": 7519, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23677607199999784, "nonpadding_tokens": 5053, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2394267199999831, "nonpadding_tokens": 4570, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23993196699996133, "nonpadding_tokens": 4835, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23342424400004802, "nonpadding_tokens": 4995, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23598617200002536, "nonpadding_tokens": 5376, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23361371399994368, "nonpadding_tokens": 6496, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2348914510000668, "nonpadding_tokens": 2541, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0006610629999386219, "prefetch": {"data_wait_seconds": 0.008772163001211197, "consumed_batches": 929, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1479, "tokens": 115762578, "loss_tokens": 47110, "loss": 3.048954906070898, "lr": 0.00029975373402785704, "optimizer_lrs": [0.019983582268523804, 0.00029975373402785704], "gradient_norm": 0.49968159198760986, "seconds": 0.2815538129999595, "nonpadding_tokens_per_second": 167321.47754648514, "ranks": [{"compute_seconds": 0.2769738490000009, "nonpadding_tokens": 5100, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27736489900007655, "nonpadding_tokens": 2785, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27465674400002626, "nonpadding_tokens": 1747, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27832211499992354, "nonpadding_tokens": 10285, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2790920460000734, "nonpadding_tokens": 9347, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2765637310000102, "nonpadding_tokens": 4332, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2790930799999387, "nonpadding_tokens": 3358, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27448289500000556, "nonpadding_tokens": 10156, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042691499993452453, "prefetch": {"data_wait_seconds": 0.008781169001238709, "consumed_batches": 930, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1480, "tokens": 115830015, "loss_tokens": 67437, "loss": 2.5660072085798595, "lr": 0.0002997533871653072, "optimizer_lrs": [0.019983559144353815, 0.0002997533871653072], "gradient_norm": 0.44422611594200134, "seconds": 0.26042333999998846, "nonpadding_tokens_per_second": 258951.44421388264, "ranks": [{"compute_seconds": 0.2513849110000592, "nonpadding_tokens": 5377, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25616548599998623, "nonpadding_tokens": 13120, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25557242100001076, "nonpadding_tokens": 5874, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2560507900000175, "nonpadding_tokens": 9809, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.2527211650000254, "nonpadding_tokens": 4285, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25340999500008365, "nonpadding_tokens": 3497, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25202097799990497, "nonpadding_tokens": 12223, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25801414900001873, "nonpadding_tokens": 13252, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003394670000034239, "prefetch": {"data_wait_seconds": 0.008790029001218613, "consumed_batches": 931, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1481, "tokens": 115895945, "loss_tokens": 65930, "loss": 2.6831878981495527, "lr": 0.00029975304781825276, "optimizer_lrs": [0.01998353652121685, 0.00029975304781825276], "gradient_norm": 0.4211573898792267, "seconds": 0.3361356829999522, "nonpadding_tokens_per_second": 196141.03272698176, "ranks": [{"compute_seconds": 0.33338948799996615, "nonpadding_tokens": 12683, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32765250900001774, "nonpadding_tokens": 3935, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3308584549999978, "nonpadding_tokens": 15273, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3322550620000584, "nonpadding_tokens": 2761, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.3300934100000177, "nonpadding_tokens": 8301, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33281889500005946, "nonpadding_tokens": 4120, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33113945499997044, "nonpadding_tokens": 6110, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33217077000006157, "nonpadding_tokens": 12747, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0006481140000005325, "prefetch": {"data_wait_seconds": 0.008798677001280453, "consumed_batches": 932, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1482, "tokens": 115941078, "loss_tokens": 45133, "loss": 2.4901440603327942, "lr": 0.0002997528153806389, "optimizer_lrs": [0.019983521025375928, 0.0002997528153806389], "gradient_norm": 0.41411206126213074, "seconds": 0.2780983650000053, "nonpadding_tokens_per_second": 162291.49711110003, "ranks": [{"compute_seconds": 0.2751301579999108, "nonpadding_tokens": 7646, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.273927169999979, "nonpadding_tokens": 14633, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2713781209999979, "nonpadding_tokens": 3198, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27572139500000503, "nonpadding_tokens": 2801, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.269997582999963, "nonpadding_tokens": 6918, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2729089209999529, "nonpadding_tokens": 4213, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27483078899990687, "nonpadding_tokens": 1735, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27552594299993416, "nonpadding_tokens": 3989, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003799609999077802, "prefetch": {"data_wait_seconds": 0.00880822300128159, "consumed_batches": 933, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1483, "tokens": 115997292, "loss_tokens": 56214, "loss": 2.425319704610951, "lr": 0.0002997525257224629, "optimizer_lrs": [0.01998350171483086, 0.0002997525257224629], "gradient_norm": 0.7941479682922363, "seconds": 0.29586090299994794, "nonpadding_tokens_per_second": 190001.44807916676, "ranks": [{"compute_seconds": 0.2912268850000146, "nonpadding_tokens": 7461, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2893660949999912, "nonpadding_tokens": 6219, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28640470400000595, "nonpadding_tokens": 2406, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28729068300003746, "nonpadding_tokens": 10067, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.29123967499992887, "nonpadding_tokens": 16076, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28842659100007495, "nonpadding_tokens": 5835, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.285540200000014, "nonpadding_tokens": 4142, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2934268650000149, "nonpadding_tokens": 4008, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000348666999911984, "prefetch": {"data_wait_seconds": 0.008817731001343532, "consumed_batches": 934, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1484, "tokens": 116060081, "loss_tokens": 62789, "loss": 3.7080400229339534, "lr": 0.00029975220198443154, "optimizer_lrs": [0.019983480132295436, 0.00029975220198443154], "gradient_norm": 3.828277111053467, "seconds": 0.5721740349999891, "nonpadding_tokens_per_second": 109737.59059164787, "ranks": [{"compute_seconds": 0.5688506210000241, "nonpadding_tokens": 4231, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5673972939999885, "nonpadding_tokens": 7669, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5685948309999276, "nonpadding_tokens": 3861, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.5650820110000723, "nonpadding_tokens": 14420, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.5696794790000013, "nonpadding_tokens": 5922, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.566155608000031, "nonpadding_tokens": 17279, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.5691978280000285, "nonpadding_tokens": 4193, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.5640210899999829, "nonpadding_tokens": 5214, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003904540000121415, "prefetch": {"data_wait_seconds": 0.008828157001360637, "consumed_batches": 935, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1485, "tokens": 116133266, "loss_tokens": 73185, "loss": 2.6035814630730343, "lr": 0.00029975182437814014, "optimizer_lrs": [0.019983454958542677, 0.00029975182437814014], "gradient_norm": 0.8803961873054504, "seconds": 0.2837069760000759, "nonpadding_tokens_per_second": 257959.81837253246, "ranks": [{"compute_seconds": 0.2784430330000305, "nonpadding_tokens": 13351, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2764427990000513, "nonpadding_tokens": 16230, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2747422689999439, "nonpadding_tokens": 8839, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28001368299999285, "nonpadding_tokens": 2535, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27990745400006745, "nonpadding_tokens": 11703, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2765455069999234, "nonpadding_tokens": 12457, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27964065900005153, "nonpadding_tokens": 2336, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28117275199997493, "nonpadding_tokens": 5734, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004468079999924157, "prefetch": {"data_wait_seconds": 0.008838521001351296, "consumed_batches": 936, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1486, "tokens": 116195324, "loss_tokens": 62058, "loss": 3.022008896919011, "lr": 0.00029975150395782546, "optimizer_lrs": [0.019983433597188367, 0.00029975150395782546], "gradient_norm": 0.583534300327301, "seconds": 0.23915769399991404, "nonpadding_tokens_per_second": 259485.693151157, "ranks": [{"compute_seconds": 0.23626235699998688, "nonpadding_tokens": 3620, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23678300900007798, "nonpadding_tokens": 3512, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23383130800004892, "nonpadding_tokens": 10286, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2341583449999689, "nonpadding_tokens": 11625, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.23286507900002107, "nonpadding_tokens": 3097, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23400592200005121, "nonpadding_tokens": 5047, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23612167999999656, "nonpadding_tokens": 20755, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23398307100001148, "nonpadding_tokens": 4116, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004241919999685706, "prefetch": {"data_wait_seconds": 0.008848887001434491, "consumed_batches": 937, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1487, "tokens": 116258073, "loss_tokens": 62749, "loss": 2.598086473489617, "lr": 0.0002997511797597262, "optimizer_lrs": [0.01998341198398175, 0.0002997511797597262], "gradient_norm": 0.4884791076183319, "seconds": 0.28165101199999754, "nonpadding_tokens_per_second": 222789.89716536345, "ranks": [{"compute_seconds": 0.2779212489999736, "nonpadding_tokens": 7650, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2781835209999599, "nonpadding_tokens": 2378, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27540220000003046, "nonpadding_tokens": 12178, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27569899100001294, "nonpadding_tokens": 11119, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17867735040}, {"compute_seconds": 0.27391320799995356, "nonpadding_tokens": 3194, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2770073419999335, "nonpadding_tokens": 5199, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27396864399997867, "nonpadding_tokens": 6103, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2791753429999062, "nonpadding_tokens": 14928, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004280969999399531, "prefetch": {"data_wait_seconds": 0.008859081001403979, "consumed_batches": 938, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1488, "tokens": 116367086, "loss_tokens": 109013, "loss": 2.41028678919028, "lr": 0.0002997506160326445, "optimizer_lrs": [0.0199833744021763, 0.0002997506160326445], "gradient_norm": 1.032921314239502, "seconds": 0.5121470000000272, "nonpadding_tokens_per_second": 212854.90298682646, "ranks": [{"compute_seconds": 0.5095446189999393, "nonpadding_tokens": 2912, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5056947509999645, "nonpadding_tokens": 4776, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5056895699999586, "nonpadding_tokens": 13796, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.5063824799999566, "nonpadding_tokens": 4774, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.5051683799999864, "nonpadding_tokens": 12336, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5069271820000267, "nonpadding_tokens": 19123, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.5045594440000514, "nonpadding_tokens": 25175, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.5096660340000199, "nonpadding_tokens": 26121, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041709799995715, "prefetch": {"data_wait_seconds": 0.008869267001344383, "consumed_batches": 939, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1489, "tokens": 116507210, "loss_tokens": 140124, "loss": 2.3627244886671805, "lr": 0.0002997498904885931, "optimizer_lrs": [0.019983326032572874, 0.0002997498904885931], "gradient_norm": 2.3495707511901855, "seconds": 0.29805968000005123, "nonpadding_tokens_per_second": 470120.6147707597, "ranks": [{"compute_seconds": 0.29206595999994533, "nonpadding_tokens": 11131, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29095052000002397, "nonpadding_tokens": 17505, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29118337900001734, "nonpadding_tokens": 25250, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2919129880000355, "nonpadding_tokens": 10442, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2903796649999322, "nonpadding_tokens": 12701, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29227576300002056, "nonpadding_tokens": 29238, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2901171629999908, "nonpadding_tokens": 14226, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2953977050000276, "nonpadding_tokens": 19631, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004251359999898341, "prefetch": {"data_wait_seconds": 0.008879459001335022, "consumed_batches": 940, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1490, "tokens": 116645093, "loss_tokens": 137883, "loss": 2.460402116286997, "lr": 0.00029974917552035783, "optimizer_lrs": [0.019983278368023858, 0.00029974917552035783], "gradient_norm": 3.898494243621826, "seconds": 0.5181394989999717, "nonpadding_tokens_per_second": 266111.7329717562, "ranks": [{"compute_seconds": 0.5144802550000804, "nonpadding_tokens": 13330, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5110009600000467, "nonpadding_tokens": 17967, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5111074940000435, "nonpadding_tokens": 5825, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.5115907130000323, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.5152096990000246, "nonpadding_tokens": 10371, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5130144500000142, "nonpadding_tokens": 13205, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.5146771319999743, "nonpadding_tokens": 22686, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.5152570090000381, "nonpadding_tokens": 21731, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041125600000668783, "prefetch": {"data_wait_seconds": 0.008888794001293832, "consumed_batches": 941, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1491, "tokens": 116694302, "loss_tokens": 49209, "loss": 2.9417792603995205, "lr": 0.0002997489201088231, "optimizer_lrs": [0.019983261340588206, 0.0002997489201088231], "gradient_norm": 0.7927699089050293, "seconds": 0.3062034639999638, "nonpadding_tokens_per_second": 160706.86907711084, "ranks": [{"compute_seconds": 0.30269101900000805, "nonpadding_tokens": 3148, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30245993800008364, "nonpadding_tokens": 3426, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29937522099999114, "nonpadding_tokens": 10133, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3029480270000704, "nonpadding_tokens": 13397, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3034418329999653, "nonpadding_tokens": 3643, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30124886299995524, "nonpadding_tokens": 6691, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3030657729999575, "nonpadding_tokens": 4105, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30375447699998404, "nonpadding_tokens": 4666, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004439260000026479, "prefetch": {"data_wait_seconds": 0.008897524001284296, "consumed_batches": 942, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1492, "tokens": 116811248, "loss_tokens": 116946, "loss": 2.333991489661895, "lr": 0.00029974831259810237, "optimizer_lrs": [0.019983220839873493, 0.00029974831259810237], "gradient_norm": 1.0157368183135986, "seconds": 0.3071136429999797, "nonpadding_tokens_per_second": 380790.6378161381, "ranks": [{"compute_seconds": 0.30443870199997036, "nonpadding_tokens": 10331, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30259601499994915, "nonpadding_tokens": 4041, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3041471480000837, "nonpadding_tokens": 15269, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30453717900002175, "nonpadding_tokens": 10956, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2988885830000072, "nonpadding_tokens": 11426, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30154281600005106, "nonpadding_tokens": 11068, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3047484749999967, "nonpadding_tokens": 28057, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29975638100006563, "nonpadding_tokens": 25798, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033752100000583596, "prefetch": {"data_wait_seconds": 0.008905265001274643, "consumed_batches": 943, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1493, "tokens": 116948763, "loss_tokens": 137515, "loss": 2.4587158673599245, "lr": 0.00029974759729738186, "optimizer_lrs": [0.01998317315315879, 0.00029974759729738186], "gradient_norm": 1.0251224040985107, "seconds": 0.34142613400001665, "nonpadding_tokens_per_second": 402766.473640806, "ranks": [{"compute_seconds": 0.3355212629999187, "nonpadding_tokens": 25442, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3346276779999471, "nonpadding_tokens": 20648, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3378458199999841, "nonpadding_tokens": 30231, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33769281299998966, "nonpadding_tokens": 21194, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33891335499993147, "nonpadding_tokens": 12430, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33693603599999733, "nonpadding_tokens": 10968, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33818403999998736, "nonpadding_tokens": 12909, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3351469149999957, "nonpadding_tokens": 3693, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045909799996479705, "prefetch": {"data_wait_seconds": 0.00891456700124138, "consumed_batches": 944, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1494, "tokens": 117038874, "loss_tokens": 90111, "loss": 2.9357368967162722, "lr": 0.0002997471280242761, "optimizer_lrs": [0.01998314186828507, 0.0002997471280242761], "gradient_norm": 0.5134189128875732, "seconds": 0.2693241859999489, "nonpadding_tokens_per_second": 334581.9079167925, "ranks": [{"compute_seconds": 0.26663419699991664, "nonpadding_tokens": 12367, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26302638300001036, "nonpadding_tokens": 10534, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2657935390000148, "nonpadding_tokens": 13545, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2669453210000938, "nonpadding_tokens": 8344, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2629615210000793, "nonpadding_tokens": 8954, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26403442699995594, "nonpadding_tokens": 17078, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26273194399993827, "nonpadding_tokens": 1993, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2628452970000126, "nonpadding_tokens": 17296, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038812399998278124, "prefetch": {"data_wait_seconds": 0.008924091001247803, "consumed_batches": 945, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1495, "tokens": 117114588, "loss_tokens": 75714, "loss": 2.482309455648889, "lr": 0.0002997467333901218, "optimizer_lrs": [0.019983115559341454, 0.0002997467333901218], "gradient_norm": 0.4810304045677185, "seconds": 0.38413260699996954, "nonpadding_tokens_per_second": 197103.80899793285, "ranks": [{"compute_seconds": 0.38158287699991433, "nonpadding_tokens": 14426, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37893418900000597, "nonpadding_tokens": 3153, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37855879500000356, "nonpadding_tokens": 11687, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3792102070000283, "nonpadding_tokens": 3477, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3774037660000431, "nonpadding_tokens": 6580, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3804606360000662, "nonpadding_tokens": 10955, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37736328000005415, "nonpadding_tokens": 8161, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37806752500000584, "nonpadding_tokens": 17275, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004285830000299029, "prefetch": {"data_wait_seconds": 0.00893496300120944, "consumed_batches": 946, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1496, "tokens": 117210019, "loss_tokens": 95431, "loss": 2.5969519207595018, "lr": 0.0002997462355497257, "optimizer_lrs": [0.019983082369981715, 0.0002997462355497257], "gradient_norm": 0.6454264521598816, "seconds": 0.3782683919999954, "nonpadding_tokens_per_second": 252283.83343221858, "ranks": [{"compute_seconds": 0.37411067400000775, "nonpadding_tokens": 25127, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37193587100000514, "nonpadding_tokens": 9219, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3709684880000168, "nonpadding_tokens": 9040, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3753650229999721, "nonpadding_tokens": 4506, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.37562139800002114, "nonpadding_tokens": 12062, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3730876070000022, "nonpadding_tokens": 4475, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37525143400000616, "nonpadding_tokens": 10128, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37587695399997756, "nonpadding_tokens": 20874, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004518230000485346, "prefetch": {"data_wait_seconds": 0.008944147001173519, "consumed_batches": 947, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1497, "tokens": 117270608, "loss_tokens": 60589, "loss": 2.788544537787387, "lr": 0.00029974591921814287, "optimizer_lrs": [0.019983061281209528, 0.00029974591921814287], "gradient_norm": 1.570170283317566, "seconds": 0.44547272799991333, "nonpadding_tokens_per_second": 136010.5707750796, "ranks": [{"compute_seconds": 0.44001262300002963, "nonpadding_tokens": 8895, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4392477039999676, "nonpadding_tokens": 4571, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4420177509999803, "nonpadding_tokens": 5546, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4421284449999803, "nonpadding_tokens": 8283, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4387171199999784, "nonpadding_tokens": 15081, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4417180290000715, "nonpadding_tokens": 3714, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4428391289999354, "nonpadding_tokens": 4475, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.44305791399995087, "nonpadding_tokens": 10024, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004514630001040132, "prefetch": {"data_wait_seconds": 0.008954043001111245, "consumed_batches": 948, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1498, "tokens": 117325025, "loss_tokens": 54417, "loss": 2.558784766249518, "lr": 0.0002997456349424107, "optimizer_lrs": [0.019983042329494046, 0.0002997456349424107], "gradient_norm": 0.4699237048625946, "seconds": 0.24374766800008274, "nonpadding_tokens_per_second": 223251.36665505054, "ranks": [{"compute_seconds": 0.23910517799993158, "nonpadding_tokens": 3284, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24053544599996712, "nonpadding_tokens": 4063, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2376280249999354, "nonpadding_tokens": 11428, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24027250500000719, "nonpadding_tokens": 16871, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2364082110000254, "nonpadding_tokens": 10137, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23806154299995796, "nonpadding_tokens": 2107, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2408492600000045, "nonpadding_tokens": 4575, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24137266900004306, "nonpadding_tokens": 1952, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003980960000262712, "prefetch": {"data_wait_seconds": 0.008963310001220748, "consumed_batches": 949, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1499, "tokens": 117424988, "loss_tokens": 99963, "loss": 2.3355238313175875, "lr": 0.0002997451123194991, "optimizer_lrs": [0.01998300748796661, 0.0002997451123194991], "gradient_norm": 0.43167227506637573, "seconds": 0.3068148059999203, "nonpadding_tokens_per_second": 325808.9181003409, "ranks": [{"compute_seconds": 0.3023414319999347, "nonpadding_tokens": 3927, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30049017099997855, "nonpadding_tokens": 8770, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30320290500003466, "nonpadding_tokens": 5852, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3033110359999682, "nonpadding_tokens": 19134, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.299979932000042, "nonpadding_tokens": 4788, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30279841100002614, "nonpadding_tokens": 21301, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3000288260000161, "nonpadding_tokens": 17044, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30447317899995596, "nonpadding_tokens": 19147, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003727360000311819, "prefetch": {"data_wait_seconds": 0.008970956001306263, "consumed_batches": 950, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1500, "tokens": 117507250, "loss_tokens": 82262, "loss": 2.484246789222241, "lr": 0.0002997446818383979, "optimizer_lrs": [0.01998297878922653, 0.0002997446818383979], "gradient_norm": 0.5798958539962769, "seconds": 0.2791003429999819, "nonpadding_tokens_per_second": 294739.8742537745, "ranks": [{"compute_seconds": 0.27684961200009184, "nonpadding_tokens": 5430, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2760552169999073, "nonpadding_tokens": 7270, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27314604399998643, "nonpadding_tokens": 4042, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2738797310000791, "nonpadding_tokens": 6979, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2718562139999676, "nonpadding_tokens": 16773, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27361389199995756, "nonpadding_tokens": 18675, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2758231039999828, "nonpadding_tokens": 11158, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27270694599997114, "nonpadding_tokens": 11935, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033709499996348313, "prefetch": {"data_wait_seconds": 0.008979152001373336, "consumed_batches": 951, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1501, "tokens": 117620273, "loss_tokens": 113023, "loss": 2.3603121709740496, "lr": 0.0002997440897916901, "optimizer_lrs": [0.01998293931944601, 0.0002997440897916901], "gradient_norm": 0.5124669671058655, "seconds": 0.40522880699995767, "nonpadding_tokens_per_second": 278911.56316537934, "ranks": [{"compute_seconds": 0.40227863600000546, "nonpadding_tokens": 21299, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3981816399999616, "nonpadding_tokens": 16739, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3973785809999981, "nonpadding_tokens": 28461, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4010181119999743, "nonpadding_tokens": 6764, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4014657449999959, "nonpadding_tokens": 2763, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3987652240000443, "nonpadding_tokens": 19625, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4010466860000861, "nonpadding_tokens": 7155, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3966167869999708, "nonpadding_tokens": 10217, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046615600001587154, "prefetch": {"data_wait_seconds": 0.008990095001379927, "consumed_batches": 952, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1502, "tokens": 117673520, "loss_tokens": 53247, "loss": 2.5830346075835258, "lr": 0.0002997438106313458, "optimizer_lrs": [0.019982920708756388, 0.0002997438106313458], "gradient_norm": 0.4671717584133148, "seconds": 0.27123189700000694, "nonpadding_tokens_per_second": 196315.4060748196, "ranks": [{"compute_seconds": 0.26871454399997674, "nonpadding_tokens": 7739, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2657145999999102, "nonpadding_tokens": 5495, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2678510790000246, "nonpadding_tokens": 4074, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2654752209999742, "nonpadding_tokens": 4122, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2688560530000359, "nonpadding_tokens": 4607, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26605572000005395, "nonpadding_tokens": 2541, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2685244919999832, "nonpadding_tokens": 11347, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26423944199996185, "nonpadding_tokens": 13322, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040074700007153297, "prefetch": {"data_wait_seconds": 0.008998614001328065, "consumed_batches": 953, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1503, "tokens": 117770466, "loss_tokens": 96946, "loss": 2.6007158624388835, "lr": 0.00029974330197797837, "optimizer_lrs": [0.01998288679853189, 0.00029974330197797837], "gradient_norm": 0.40064409375190735, "seconds": 0.29581423000001905, "nonpadding_tokens_per_second": 327725.9515202962, "ranks": [{"compute_seconds": 0.2888531600000306, "nonpadding_tokens": 2796, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29092491799997333, "nonpadding_tokens": 5735, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29187380999997004, "nonpadding_tokens": 24612, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2886120429999437, "nonpadding_tokens": 19731, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28729753999994045, "nonpadding_tokens": 5955, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2900971889999937, "nonpadding_tokens": 17390, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28716625000004115, "nonpadding_tokens": 13035, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29342555000005177, "nonpadding_tokens": 7692, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003878500000382701, "prefetch": {"data_wait_seconds": 0.009008709001321336, "consumed_batches": 954, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1504, "tokens": 117833488, "loss_tokens": 63022, "loss": 2.8672646060105995, "lr": 0.00029974297104573886, "optimizer_lrs": [0.019982864736382593, 0.00029974297104573886], "gradient_norm": 0.49607107043266296, "seconds": 0.3554766180000115, "nonpadding_tokens_per_second": 177288.7352045134, "ranks": [{"compute_seconds": 0.35005025899999964, "nonpadding_tokens": 16103, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3491240309999739, "nonpadding_tokens": 4313, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3517917970000326, "nonpadding_tokens": 13571, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.35201954699994076, "nonpadding_tokens": 6295, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3526774580000165, "nonpadding_tokens": 6541, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3505478839999796, "nonpadding_tokens": 5279, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3474761379999336, "nonpadding_tokens": 5619, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35310882399994625, "nonpadding_tokens": 5301, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003456980000464682, "prefetch": {"data_wait_seconds": 0.009017895001264264, "consumed_batches": 955, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1505, "tokens": 117904846, "loss_tokens": 71358, "loss": 2.689576457790297, "lr": 0.0002997425960835762, "optimizer_lrs": [0.01998283973890508, 0.0002997425960835762], "gradient_norm": 0.4347388744354248, "seconds": 0.34567326700005196, "nonpadding_tokens_per_second": 206431.9309945055, "ranks": [{"compute_seconds": 0.3424262759999692, "nonpadding_tokens": 14823, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3392681459999949, "nonpadding_tokens": 13669, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34254154199993536, "nonpadding_tokens": 4837, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33887258899994777, "nonpadding_tokens": 7367, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34305520299994896, "nonpadding_tokens": 3760, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3402861900000289, "nonpadding_tokens": 9370, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3427827120000302, "nonpadding_tokens": 9482, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34327457899996716, "nonpadding_tokens": 8050, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037427699999170727, "prefetch": {"data_wait_seconds": 0.009027216001257443, "consumed_batches": 956, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1506, "tokens": 117960217, "loss_tokens": 55371, "loss": 2.63453680627043, "lr": 0.000299742304939556, "optimizer_lrs": [0.019982820329303735, 0.000299742304939556], "gradient_norm": 0.46810290217399597, "seconds": 0.32032428299999083, "nonpadding_tokens_per_second": 172859.20218543528, "ranks": [{"compute_seconds": 0.3165160309999919, "nonpadding_tokens": 3292, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3168889549999676, "nonpadding_tokens": 8209, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31395330999998805, "nonpadding_tokens": 2969, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31454061900001307, "nonpadding_tokens": 6611, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3178622529999302, "nonpadding_tokens": 4023, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31540567599995484, "nonpadding_tokens": 9309, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3134737379999706, "nonpadding_tokens": 9938, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31363934600005905, "nonpadding_tokens": 11020, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004457549999870025, "prefetch": {"data_wait_seconds": 0.009038180001269325, "consumed_batches": 957, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1507, "tokens": 118031952, "loss_tokens": 71735, "loss": 2.255503328221928, "lr": 0.0002997419275081854, "optimizer_lrs": [0.01998279516721236, 0.0002997419275081854], "gradient_norm": 1.2230809926986694, "seconds": 0.3124607439999636, "nonpadding_tokens_per_second": 229580.8397614529, "ranks": [{"compute_seconds": 0.30853058999991845, "nonpadding_tokens": 6840, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30870358000004217, "nonpadding_tokens": 8496, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3062623560000475, "nonpadding_tokens": 3386, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3099700010000106, "nonpadding_tokens": 15406, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30510137700002815, "nonpadding_tokens": 10391, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3076038440000275, "nonpadding_tokens": 10082, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30481809799994153, "nonpadding_tokens": 15393, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.305166047000057, "nonpadding_tokens": 1741, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004107510000039838, "prefetch": {"data_wait_seconds": 0.00904749300123342, "consumed_batches": 958, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1508, "tokens": 118103821, "loss_tokens": 71869, "loss": 2.550575091485898, "lr": 0.00029974154909507005, "optimizer_lrs": [0.019982769939671338, 0.00029974154909507005], "gradient_norm": 0.41948074102401733, "seconds": 0.3808943779999936, "nonpadding_tokens_per_second": 188684.85373129137, "ranks": [{"compute_seconds": 0.37608315299996775, "nonpadding_tokens": 10038, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3771621730001016, "nonpadding_tokens": 3508, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37431056000002627, "nonpadding_tokens": 5956, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.37695017099997585, "nonpadding_tokens": 3330, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3777461829999993, "nonpadding_tokens": 6044, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3761593130000165, "nonpadding_tokens": 10674, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37324396100007107, "nonpadding_tokens": 12227, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37852960600002916, "nonpadding_tokens": 20092, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00048044199991181813, "prefetch": {"data_wait_seconds": 0.009057081001245137, "consumed_batches": 959, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1509, "tokens": 118165941, "loss_tokens": 62120, "loss": 2.346637556342563, "lr": 0.0002997412217903782, "optimizer_lrs": [0.019982748119358545, 0.0002997412217903782], "gradient_norm": 0.4661179780960083, "seconds": 0.24835879199997635, "nonpadding_tokens_per_second": 250122.00896840374, "ranks": [{"compute_seconds": 0.24450241900001402, "nonpadding_tokens": 3997, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2453601450000633, "nonpadding_tokens": 7280, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24221768100005647, "nonpadding_tokens": 6977, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24302161000002798, "nonpadding_tokens": 10238, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2459438879999425, "nonpadding_tokens": 9834, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24399833899997247, "nonpadding_tokens": 4453, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24602827100000013, "nonpadding_tokens": 9181, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24177282700009073, "nonpadding_tokens": 10160, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037082400001509086, "prefetch": {"data_wait_seconds": 0.009065913001180093, "consumed_batches": 960, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1510, "tokens": 118251847, "loss_tokens": 85906, "loss": 2.6560057286452636, "lr": 0.000299740768818456, "optimizer_lrs": [0.0199827179212304, 0.000299740768818456], "gradient_norm": 0.4056888222694397, "seconds": 0.33098426300000483, "nonpadding_tokens_per_second": 259547.08305874575, "ranks": [{"compute_seconds": 0.3248886330000005, "nonpadding_tokens": 11866, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3218179489999784, "nonpadding_tokens": 5030, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3222690530000136, "nonpadding_tokens": 2834, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32517705199995817, "nonpadding_tokens": 19230, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3263943260000133, "nonpadding_tokens": 14785, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32406430399998953, "nonpadding_tokens": 12181, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32202713599997423, "nonpadding_tokens": 7834, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3262690530000327, "nonpadding_tokens": 12146, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.002480834000039067, "prefetch": {"data_wait_seconds": 0.009073381001101666, "consumed_batches": 961, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1511, "tokens": 118330079, "loss_tokens": 78232, "loss": 2.584827620091523, "lr": 0.0002997403559663318, "optimizer_lrs": [0.019982690397755458, 0.0002997403559663318], "gradient_norm": 0.4017692804336548, "seconds": 0.35244808199990985, "nonpadding_tokens_per_second": 221967.44427174953, "ranks": [{"compute_seconds": 0.349912677999896, "nonpadding_tokens": 15964, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3474416919999612, "nonpadding_tokens": 15294, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3494077240000024, "nonpadding_tokens": 4688, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34999860899995383, "nonpadding_tokens": 7510, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34670501100004003, "nonpadding_tokens": 5682, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34903587799999514, "nonpadding_tokens": 13322, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3459599459999936, "nonpadding_tokens": 7550, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34653378100006194, "nonpadding_tokens": 8222, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040480299992395885, "prefetch": {"data_wait_seconds": 0.009083085001137192, "consumed_batches": 962, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1512, "tokens": 118417833, "loss_tokens": 87754, "loss": 2.098484648847916, "lr": 0.0002997398924734295, "optimizer_lrs": [0.019982659498228635, 0.0002997398924734295], "gradient_norm": 0.3830207288265228, "seconds": 0.272629646999917, "nonpadding_tokens_per_second": 321879.88711303554, "ranks": [{"compute_seconds": 0.24211328700005197, "nonpadding_tokens": 17314, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23881382299998677, "nonpadding_tokens": 15617, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24110254699996858, "nonpadding_tokens": 8921, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24164948100008132, "nonpadding_tokens": 5240, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23780607599996983, "nonpadding_tokens": 12510, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24021114699996815, "nonpadding_tokens": 12070, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24161068399996566, "nonpadding_tokens": 9454, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23759136099999978, "nonpadding_tokens": 6628, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.025824305999890385, "prefetch": {"data_wait_seconds": 0.009092325001077484, "consumed_batches": 963, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1513, "tokens": 118483267, "loss_tokens": 65434, "loss": 2.449412289482532, "lr": 0.00029973954659999057, "optimizer_lrs": [0.019982636439999372, 0.00029973954659999057], "gradient_norm": 0.4023086726665497, "seconds": 0.41024511300008726, "nonpadding_tokens_per_second": 159499.76715502297, "ranks": [{"compute_seconds": 0.404587451999987, "nonpadding_tokens": 10971, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4039088780000384, "nonpadding_tokens": 6058, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.40212468499998977, "nonpadding_tokens": 9830, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4027900610000188, "nonpadding_tokens": 13700, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.40096589199993105, "nonpadding_tokens": 3826, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.40504158500004905, "nonpadding_tokens": 3930, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.40780534300006366, "nonpadding_tokens": 5332, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.40202999400003137, "nonpadding_tokens": 11787, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003425799999376977, "prefetch": {"data_wait_seconds": 0.009101287001115566, "consumed_batches": 964, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1514, "tokens": 118564572, "loss_tokens": 81305, "loss": 2.6698692423590185, "lr": 0.00029973911651517076, "optimizer_lrs": [0.019982607767678053, 0.00029973911651517076], "gradient_norm": 0.7714953422546387, "seconds": 0.29440962000001036, "nonpadding_tokens_per_second": 276162.85092857067, "ranks": [{"compute_seconds": 0.29216247799990924, "nonpadding_tokens": 3252, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2890556859999833, "nonpadding_tokens": 9461, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2883141569999452, "nonpadding_tokens": 18515, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2919805730000462, "nonpadding_tokens": 13634, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2873058369999626, "nonpadding_tokens": 18849, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28984213100000034, "nonpadding_tokens": 3708, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2873529660000713, "nonpadding_tokens": 9466, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28779119300008915, "nonpadding_tokens": 4420, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003808719999369714, "prefetch": {"data_wait_seconds": 0.009109699001172089, "consumed_batches": 965, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1515, "tokens": 118623935, "loss_tokens": 59363, "loss": 2.6867948574869867, "lr": 0.00029973880227462273, "optimizer_lrs": [0.019982586818308185, 0.00029973880227462273], "gradient_norm": 0.7054750323295593, "seconds": 0.29268205599998964, "nonpadding_tokens_per_second": 202824.18680290433, "ranks": [{"compute_seconds": 0.2899034719999918, "nonpadding_tokens": 8821, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28735277399994175, "nonpadding_tokens": 15185, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28927511799997774, "nonpadding_tokens": 2556, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28999713999996857, "nonpadding_tokens": 4012, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29030829400005587, "nonpadding_tokens": 12841, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28790774100002636, "nonpadding_tokens": 7325, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28503028000000086, "nonpadding_tokens": 3502, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28565572699994846, "nonpadding_tokens": 5121, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003823800000191113, "prefetch": {"data_wait_seconds": 0.009119415001194398, "consumed_batches": 966, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1516, "tokens": 118677303, "loss_tokens": 53368, "loss": 2.567105112145855, "lr": 0.00029973851960756284, "optimizer_lrs": [0.019982567973837524, 0.00029973851960756284], "gradient_norm": 0.564765453338623, "seconds": 0.29561933299999055, "nonpadding_tokens_per_second": 180529.4648980272, "ranks": [{"compute_seconds": 0.2902475509999931, "nonpadding_tokens": 12919, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29256483200003913, "nonpadding_tokens": 11164, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2920071979999648, "nonpadding_tokens": 2231, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2926737110000204, "nonpadding_tokens": 6795, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28849009500004286, "nonpadding_tokens": 5022, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2912994419999677, "nonpadding_tokens": 3679, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28793241700009276, "nonpadding_tokens": 4641, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2932661659999667, "nonpadding_tokens": 6917, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038595399996665947, "prefetch": {"data_wait_seconds": 0.009130178001171885, "consumed_batches": 967, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1517, "tokens": 118761378, "loss_tokens": 84075, "loss": 2.4438797204876597, "lr": 0.0002997380739890858, "optimizer_lrs": [0.019982538265939057, 0.0002997380739890858], "gradient_norm": 0.49092066287994385, "seconds": 0.32151662399996894, "nonpadding_tokens_per_second": 261495.03236886478, "ranks": [{"compute_seconds": 0.3188187169999992, "nonpadding_tokens": 5001, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3171984320000547, "nonpadding_tokens": 19898, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31487965700000586, "nonpadding_tokens": 4947, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31515062599999055, "nonpadding_tokens": 12566, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31915655900002093, "nonpadding_tokens": 15558, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3166848140000411, "nonpadding_tokens": 7627, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.319040561999941, "nonpadding_tokens": 5395, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31449101400005475, "nonpadding_tokens": 13083, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003931799999463692, "prefetch": {"data_wait_seconds": 0.009139607001273475, "consumed_batches": 968, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1518, "tokens": 118833192, "loss_tokens": 71814, "loss": 2.5124370770323337, "lr": 0.0002997376930568265, "optimizer_lrs": [0.0199825128704551, 0.0002997376930568265], "gradient_norm": 0.3725043833255768, "seconds": 0.4504159430000527, "nonpadding_tokens_per_second": 159439.29409264182, "ranks": [{"compute_seconds": 0.44774551599994084, "nonpadding_tokens": 25083, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.44388385699994615, "nonpadding_tokens": 8874, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4441406949999873, "nonpadding_tokens": 1595, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.44720451499995306, "nonpadding_tokens": 4511, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.447807508999972, "nonpadding_tokens": 4752, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4450643110000101, "nonpadding_tokens": 20372, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.44748131200003627, "nonpadding_tokens": 2368, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.443910065999944, "nonpadding_tokens": 4259, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042790599991349154, "prefetch": {"data_wait_seconds": 0.00914968200125088, "consumed_batches": 969, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1519, "tokens": 118906389, "loss_tokens": 73197, "loss": 2.666663464691176, "lr": 0.00029973730450396854, "optimizer_lrs": [0.019982486966931237, 0.00029973730450396854], "gradient_norm": 0.48748159408569336, "seconds": 0.25319672800003445, "nonpadding_tokens_per_second": 289091.41353513085, "ranks": [{"compute_seconds": 0.25074604500002806, "nonpadding_tokens": 1682, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2506790139999566, "nonpadding_tokens": 18180, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24709424300010596, "nonpadding_tokens": 6439, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2479074689999834, "nonpadding_tokens": 3325, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24659834800002045, "nonpadding_tokens": 11931, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2494287480000139, "nonpadding_tokens": 5616, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24624132200005988, "nonpadding_tokens": 19734, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24721317800003817, "nonpadding_tokens": 6290, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003725719999465582, "prefetch": {"data_wait_seconds": 0.00915801800120164, "consumed_batches": 970, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1520, "tokens": 118967766, "loss_tokens": 61377, "loss": 2.3680735454649136, "lr": 0.00029973697847396905, "optimizer_lrs": [0.019982465231597937, 0.00029973697847396905], "gradient_norm": 0.47894927859306335, "seconds": 0.371501348000038, "nonpadding_tokens_per_second": 165213.39782593123, "ranks": [{"compute_seconds": 0.36711329799993564, "nonpadding_tokens": 4398, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3674088039999788, "nonpadding_tokens": 3198, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3676841610000565, "nonpadding_tokens": 2826, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3656097220000447, "nonpadding_tokens": 10954, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3687154770000234, "nonpadding_tokens": 12650, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36633947099994657, "nonpadding_tokens": 18209, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36850043699996604, "nonpadding_tokens": 7071, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36907533999999487, "nonpadding_tokens": 2071, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035094600002594234, "prefetch": {"data_wait_seconds": 0.009167164001112837, "consumed_batches": 971, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1521, "tokens": 119021404, "loss_tokens": 53638, "loss": 2.8553287781050747, "lr": 0.00029973669338755846, "optimizer_lrs": [0.019982446225837232, 0.00029973669338755846], "gradient_norm": 0.43346914649009705, "seconds": 0.32512675400005264, "nonpadding_tokens_per_second": 164975.65746309306, "ranks": [{"compute_seconds": 0.32232092100002774, "nonpadding_tokens": 12772, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32039293000002544, "nonpadding_tokens": 9511, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3175485720000779, "nonpadding_tokens": 3648, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3216407980000895, "nonpadding_tokens": 2229, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3165807739999309, "nonpadding_tokens": 12149, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32028428500007067, "nonpadding_tokens": 3622, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3226856599999337, "nonpadding_tokens": 6865, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31706944399991244, "nonpadding_tokens": 2842, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003941969999914363, "prefetch": {"data_wait_seconds": 0.009176480001158893, "consumed_batches": 972, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1522, "tokens": 119095877, "loss_tokens": 74473, "loss": 2.519889968512078, "lr": 0.00029973629730716266, "optimizer_lrs": [0.019982419820477514, 0.00029973629730716266], "gradient_norm": 0.46179911494255066, "seconds": 0.26253751299998385, "nonpadding_tokens_per_second": 283666.1288857588, "ranks": [{"compute_seconds": 0.2597780910000438, "nonpadding_tokens": 3787, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2574862449999955, "nonpadding_tokens": 12134, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.254805092999959, "nonpadding_tokens": 4306, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2593585490000123, "nonpadding_tokens": 7085, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2601263649999055, "nonpadding_tokens": 4102, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2569752609999796, "nonpadding_tokens": 13704, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2539774419999503, "nonpadding_tokens": 20094, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2544044720000329, "nonpadding_tokens": 9261, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039141500008099683, "prefetch": {"data_wait_seconds": 0.009187194001199828, "consumed_batches": 973, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1523, "tokens": 119144424, "loss_tokens": 48547, "loss": 2.585029552547016, "lr": 0.0002997360389526009, "optimizer_lrs": [0.01998240259684006, 0.0002997360389526009], "gradient_norm": 0.42444419860839844, "seconds": 0.23852388000000246, "nonpadding_tokens_per_second": 203530.98398365604, "ranks": [{"compute_seconds": 0.23576811800000996, "nonpadding_tokens": 3282, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23292911199996524, "nonpadding_tokens": 4081, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23176470299995344, "nonpadding_tokens": 2503, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23567792700009704, "nonpadding_tokens": 2025, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23615997199999583, "nonpadding_tokens": 8750, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23378995099994881, "nonpadding_tokens": 12441, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23117437000007612, "nonpadding_tokens": 11614, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23158181700000569, "nonpadding_tokens": 3851, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041588799990677217, "prefetch": {"data_wait_seconds": 0.009197901001201103, "consumed_batches": 974, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1524, "tokens": 119253293, "loss_tokens": 108869, "loss": 2.726699634423022, "lr": 0.000299735459120517, "optimizer_lrs": [0.019982363941367803, 0.000299735459120517], "gradient_norm": 0.4293498992919922, "seconds": 0.33481193000000076, "nonpadding_tokens_per_second": 325164.6379506243, "ranks": [{"compute_seconds": 0.3305787939999618, "nonpadding_tokens": 27503, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3284849320000376, "nonpadding_tokens": 17458, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3313763230000859, "nonpadding_tokens": 13206, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32667859100001806, "nonpadding_tokens": 16209, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3254971100000148, "nonpadding_tokens": 10501, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3277971750000006, "nonpadding_tokens": 6090, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32535319700002674, "nonpadding_tokens": 6790, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33247427699996024, "nonpadding_tokens": 11112, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040335599999252736, "prefetch": {"data_wait_seconds": 0.009207450001170514, "consumed_batches": 975, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1525, "tokens": 119322342, "loss_tokens": 69049, "loss": 2.533944191805819, "lr": 0.0002997350910388535, "optimizer_lrs": [0.019982339402590237, 0.0002997350910388535], "gradient_norm": 0.6179911494255066, "seconds": 0.38321682799994505, "nonpadding_tokens_per_second": 180182.5884327029, "ranks": [{"compute_seconds": 0.37769621699999334, "nonpadding_tokens": 11982, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3808906739999429, "nonpadding_tokens": 4466, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37943081699995673, "nonpadding_tokens": 16459, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3778216389999898, "nonpadding_tokens": 4016, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3808848769999713, "nonpadding_tokens": 9114, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37795655100001113, "nonpadding_tokens": 10908, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37614810299999135, "nonpadding_tokens": 7857, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37672314600001755, "nonpadding_tokens": 4247, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003594460000613253, "prefetch": {"data_wait_seconds": 0.009217168001171672, "consumed_batches": 976, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1526, "tokens": 119375080, "loss_tokens": 52738, "loss": 2.4835018748340856, "lr": 0.0002997348097345544, "optimizer_lrs": [0.019982320648970292, 0.0002997348097345544], "gradient_norm": 0.4060406982898712, "seconds": 0.3050822259999677, "nonpadding_tokens_per_second": 172864.87217385645, "ranks": [{"compute_seconds": 0.3000013780000472, "nonpadding_tokens": 16757, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29758712399996057, "nonpadding_tokens": 4295, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30001761900007295, "nonpadding_tokens": 9342, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29538949300001605, "nonpadding_tokens": 4897, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3008956070000295, "nonpadding_tokens": 4149, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29654620500002693, "nonpadding_tokens": 8056, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29324311400000624, "nonpadding_tokens": 1953, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.302607766000051, "nonpadding_tokens": 3289, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003867709999667568, "prefetch": {"data_wait_seconds": 0.009225554001162095, "consumed_batches": 977, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1527, "tokens": 119435897, "loss_tokens": 60817, "loss": 2.6901462584474736, "lr": 0.00029973448515174946, "optimizer_lrs": [0.01998229901011663, 0.00029973448515174946], "gradient_norm": 0.39541319012641907, "seconds": 0.3540993060000801, "nonpadding_tokens_per_second": 171751.25443478345, "ranks": [{"compute_seconds": 0.3499233949999052, "nonpadding_tokens": 9933, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34686792499996955, "nonpadding_tokens": 16028, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34545797600003425, "nonpadding_tokens": 8677, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3459292129999767, "nonpadding_tokens": 5378, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3505573590000495, "nonpadding_tokens": 3503, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34834901799990803, "nonpadding_tokens": 10042, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3445611409999856, "nonpadding_tokens": 3791, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35152010199999495, "nonpadding_tokens": 3465, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043135299995356036, "prefetch": {"data_wait_seconds": 0.009235386001094525, "consumed_batches": 978, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1528, "tokens": 119479019, "loss_tokens": 43122, "loss": 2.8187022430545894, "lr": 0.0002997342548877179, "optimizer_lrs": [0.019982283659181192, 0.0002997342548877179], "gradient_norm": 0.6896596550941467, "seconds": 0.28029147200004445, "nonpadding_tokens_per_second": 153846.9925335194, "ranks": [{"compute_seconds": 0.27775275900000906, "nonpadding_tokens": 5173, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27463824900007694, "nonpadding_tokens": 2164, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27664511099999345, "nonpadding_tokens": 4546, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2769919749999872, "nonpadding_tokens": 11320, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27191094499994506, "nonpadding_tokens": 3175, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2746761930000048, "nonpadding_tokens": 5487, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27222242699997423, "nonpadding_tokens": 8805, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2778561970000055, "nonpadding_tokens": 2452, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037157699989620596, "prefetch": {"data_wait_seconds": 0.009244861001050708, "consumed_batches": 979, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1529, "tokens": 119574900, "loss_tokens": 95881, "loss": 2.522922946152001, "lr": 0.000299733742542429, "optimizer_lrs": [0.019982249502828602, 0.000299733742542429], "gradient_norm": 0.47255799174308777, "seconds": 0.2671381269999529, "nonpadding_tokens_per_second": 358919.1893975393, "ranks": [{"compute_seconds": 0.26268433699999605, "nonpadding_tokens": 2018, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2642722959999446, "nonpadding_tokens": 18692, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26318118999995477, "nonpadding_tokens": 15104, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26320469399991, "nonpadding_tokens": 2936, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26416761799998767, "nonpadding_tokens": 2311, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2627427300000136, "nonpadding_tokens": 24999, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25973380699997506, "nonpadding_tokens": 19582, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.264832399999932, "nonpadding_tokens": 10239, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039513399997304077, "prefetch": {"data_wait_seconds": 0.009254317001094137, "consumed_batches": 980, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1530, "tokens": 119625557, "loss_tokens": 50657, "loss": 2.669699214817301, "lr": 0.0002997334716550093, "optimizer_lrs": [0.01998223144366729, 0.0002997334716550093], "gradient_norm": 0.46826744079589844, "seconds": 0.2582991749999337, "nonpadding_tokens_per_second": 196117.5447037839, "ranks": [{"compute_seconds": 0.2546949689999565, "nonpadding_tokens": 5609, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2518974019999405, "nonpadding_tokens": 2240, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2520904400000745, "nonpadding_tokens": 2396, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25487953800006835, "nonpadding_tokens": 9737, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2556287219999831, "nonpadding_tokens": 3228, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2529948079999258, "nonpadding_tokens": 4072, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25569724199999655, "nonpadding_tokens": 17280, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.255901693999931, "nonpadding_tokens": 6095, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039646200002607657, "prefetch": {"data_wait_seconds": 0.009264623001129735, "consumed_batches": 981, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1531, "tokens": 119684553, "loss_tokens": 58996, "loss": 2.720125887772052, "lr": 0.00029973315600150116, "optimizer_lrs": [0.01998221040010008, 0.00029973315600150116], "gradient_norm": 0.4592517018318176, "seconds": 0.28167980500006706, "nonpadding_tokens_per_second": 209443.4849526609, "ranks": [{"compute_seconds": 0.27809426900000744, "nonpadding_tokens": 1842, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2749635129999888, "nonpadding_tokens": 13138, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27488228799995795, "nonpadding_tokens": 4593, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27614727399998173, "nonpadding_tokens": 5167, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27822405699998853, "nonpadding_tokens": 10332, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2760327400000051, "nonpadding_tokens": 10520, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2741056039999421, "nonpadding_tokens": 9632, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27919115700001385, "nonpadding_tokens": 3772, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003862510000089969, "prefetch": {"data_wait_seconds": 0.009273786001088524, "consumed_batches": 982, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1532, "tokens": 119766572, "loss_tokens": 82019, "loss": 2.3217861486362916, "lr": 0.00029973271685514874, "optimizer_lrs": [0.019982181123676584, 0.00029973271685514874], "gradient_norm": 0.4989509880542755, "seconds": 0.3905349039999919, "nonpadding_tokens_per_second": 210017.0795489299, "ranks": [{"compute_seconds": 0.3864295949999814, "nonpadding_tokens": 14465, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3834391110000297, "nonpadding_tokens": 18778, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3863519039999801, "nonpadding_tokens": 11606, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.38461801999994805, "nonpadding_tokens": 10238, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.38729107500000737, "nonpadding_tokens": 3877, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.385659654000051, "nonpadding_tokens": 8263, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.382838023999966, "nonpadding_tokens": 13004, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3878530749999527, "nonpadding_tokens": 1788, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0006395039999915753, "prefetch": {"data_wait_seconds": 0.009281380001155526, "consumed_batches": 983, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1533, "tokens": 119830668, "loss_tokens": 64096, "loss": 2.7873709455348226, "lr": 0.0002997323734210928, "optimizer_lrs": [0.019982158228072856, 0.0002997323734210928], "gradient_norm": 0.43829742074012756, "seconds": 0.3473949290000746, "nonpadding_tokens_per_second": 184504.70818469036, "ranks": [{"compute_seconds": 0.3091971400000375, "nonpadding_tokens": 2844, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33847949999994853, "nonpadding_tokens": 14091, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33852711999998064, "nonpadding_tokens": 4047, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3360770520000642, "nonpadding_tokens": 12604, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3346729620000133, "nonpadding_tokens": 7259, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3375927239999328, "nonpadding_tokens": 11702, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33449388799999724, "nonpadding_tokens": 6874, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3398714810000456, "nonpadding_tokens": 4675, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043718599999920116, "prefetch": {"data_wait_seconds": 0.009290297001214753, "consumed_batches": 984, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1534, "tokens": 119882910, "loss_tokens": 52242, "loss": 2.5682595540944066, "lr": 0.0002997320933392519, "optimizer_lrs": [0.01998213955595013, 0.0002997320933392519], "gradient_norm": 0.46498996019363403, "seconds": 0.2820337709999876, "nonpadding_tokens_per_second": 185233.13649556632, "ranks": [{"compute_seconds": 0.27957080800001677, "nonpadding_tokens": 10198, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.275892143999954, "nonpadding_tokens": 18531, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27419575400006124, "nonpadding_tokens": 4710, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2787095549999776, "nonpadding_tokens": 1468, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2735085120000349, "nonpadding_tokens": 3312, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27629182599991964, "nonpadding_tokens": 3619, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27871003999996447, "nonpadding_tokens": 8373, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2739566699999614, "nonpadding_tokens": 2031, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038084100003743515, "prefetch": {"data_wait_seconds": 0.009301032001147291, "consumed_batches": 985, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1535, "tokens": 119939537, "loss_tokens": 56627, "loss": 2.5008791080226747, "lr": 0.0002997317895831026, "optimizer_lrs": [0.019982119305540173, 0.0002997317895831026], "gradient_norm": 0.4568061828613281, "seconds": 0.3169753900000387, "nonpadding_tokens_per_second": 178647.9385670701, "ranks": [{"compute_seconds": 0.31394840799998747, "nonpadding_tokens": 9859, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31007732299997315, "nonpadding_tokens": 8323, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3106920640000226, "nonpadding_tokens": 9180, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.313694258000055, "nonpadding_tokens": 4332, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3142621089999693, "nonpadding_tokens": 3372, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3122801700000082, "nonpadding_tokens": 3557, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3098597680000239, "nonpadding_tokens": 8590, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3145194429999947, "nonpadding_tokens": 9414, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040578599998752907, "prefetch": {"data_wait_seconds": 0.009310702001087634, "consumed_batches": 986, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1536, "tokens": 120027082, "loss_tokens": 87545, "loss": 2.479047204294934, "lr": 0.00029973131963955915, "optimizer_lrs": [0.019982087975970612, 0.00029973131963955915], "gradient_norm": 0.526983380317688, "seconds": 0.3085390889999644, "nonpadding_tokens_per_second": 283740.38532281434, "ranks": [{"compute_seconds": 0.30470368899989353, "nonpadding_tokens": 11057, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3043006929999592, "nonpadding_tokens": 6222, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30121400999996695, "nonpadding_tokens": 4298, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3016100739999956, "nonpadding_tokens": 18123, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3052838009999732, "nonpadding_tokens": 10863, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3027349660000027, "nonpadding_tokens": 10896, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29936836299998504, "nonpadding_tokens": 12143, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30601135599999907, "nonpadding_tokens": 13943, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004279480000377589, "prefetch": {"data_wait_seconds": 0.0093205360011126, "consumed_batches": 987, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1537, "tokens": 120099313, "loss_tokens": 72231, "loss": 2.751840665711398, "lr": 0.0002997309315925185, "optimizer_lrs": [0.019982062106167903, 0.0002997309315925185], "gradient_norm": 0.4203694760799408, "seconds": 0.3012281129999792, "nonpadding_tokens_per_second": 239788.37592759807, "ranks": [{"compute_seconds": 0.29808278200005134, "nonpadding_tokens": 18048, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2965746989999616, "nonpadding_tokens": 7330, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2974087989999816, "nonpadding_tokens": 13655, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2978636650000226, "nonpadding_tokens": 3245, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29252656500000285, "nonpadding_tokens": 15830, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29538378999995984, "nonpadding_tokens": 2663, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29262897399996746, "nonpadding_tokens": 3094, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2988559760000271, "nonpadding_tokens": 8366, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003527970000050118, "prefetch": {"data_wait_seconds": 0.009329178001124205, "consumed_batches": 988, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1538, "tokens": 120159214, "loss_tokens": 59901, "loss": 2.563157595449158, "lr": 0.000299730609573824, "optimizer_lrs": [0.019982040638254935, 0.000299730609573824], "gradient_norm": 0.4934248626232147, "seconds": 0.2608126390000507, "nonpadding_tokens_per_second": 229670.6180714975, "ranks": [{"compute_seconds": 0.2540267990000302, "nonpadding_tokens": 2521, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25644605299999057, "nonpadding_tokens": 5970, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2533901680000099, "nonpadding_tokens": 5930, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2583036659999607, "nonpadding_tokens": 3833, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25758549399995445, "nonpadding_tokens": 14123, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.255131246000019, "nonpadding_tokens": 10603, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25757195500000307, "nonpadding_tokens": 11127, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25816446499993617, "nonpadding_tokens": 5794, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003720759999623624, "prefetch": {"data_wait_seconds": 0.009338467001114736, "consumed_batches": 989, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1539, "tokens": 120219939, "loss_tokens": 60725, "loss": 2.660245728694936, "lr": 0.0002997302829290556, "optimizer_lrs": [0.019982018861937043, 0.0002997302829290556], "gradient_norm": 1.0256893634796143, "seconds": 0.30670801499991285, "nonpadding_tokens_per_second": 197989.60910759785, "ranks": [{"compute_seconds": 0.3029439580000144, "nonpadding_tokens": 4188, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2999188039999581, "nonpadding_tokens": 12360, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29735430999994605, "nonpadding_tokens": 11395, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3030732210000906, "nonpadding_tokens": 4178, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2964291969999522, "nonpadding_tokens": 4896, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2997512929999857, "nonpadding_tokens": 13607, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29643190299998423, "nonpadding_tokens": 2920, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30419044799998574, "nonpadding_tokens": 7181, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036859600004390813, "prefetch": {"data_wait_seconds": 0.009347203001155435, "consumed_batches": 990, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1540, "tokens": 120273007, "loss_tokens": 53068, "loss": 2.5228609990955, "lr": 0.00029972999731002817, "optimizer_lrs": [0.019981999820668547, 0.00029972999731002817], "gradient_norm": 0.9233938455581665, "seconds": 0.282806058999995, "nonpadding_tokens_per_second": 187648.030553691, "ranks": [{"compute_seconds": 0.27898179099997833, "nonpadding_tokens": 4857, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2791724650001015, "nonpadding_tokens": 10603, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2800700180000604, "nonpadding_tokens": 3637, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2803503970000065, "nonpadding_tokens": 1940, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27425083100001757, "nonpadding_tokens": 4865, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27695648799999617, "nonpadding_tokens": 3171, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2803865529999712, "nonpadding_tokens": 10630, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2747692510000661, "nonpadding_tokens": 13365, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043032400003539806, "prefetch": {"data_wait_seconds": 0.00935661400122001, "consumed_batches": 991, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1541, "tokens": 120329004, "loss_tokens": 55997, "loss": 2.610543310355912, "lr": 0.0002997296957630046, "optimizer_lrs": [0.01998197971753364, 0.0002997296957630046], "gradient_norm": 0.42402103543281555, "seconds": 0.32607063600005404, "nonpadding_tokens_per_second": 171732.7284876726, "ranks": [{"compute_seconds": 0.32172674200000984, "nonpadding_tokens": 13259, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3194584799999802, "nonpadding_tokens": 9307, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3205849749999743, "nonpadding_tokens": 7684, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31723412500002723, "nonpadding_tokens": 3261, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3217801989999316, "nonpadding_tokens": 6958, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3191325619999361, "nonpadding_tokens": 2542, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31563506700001653, "nonpadding_tokens": 3857, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.323617205000005, "nonpadding_tokens": 9129, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034562400003324, "prefetch": {"data_wait_seconds": 0.009366408001213244, "consumed_batches": 992, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1542, "tokens": 120379808, "loss_tokens": 50804, "loss": 2.2907169895579087, "lr": 0.0002997294220351194, "optimizer_lrs": [0.01998196146900796, 0.0002997294220351194], "gradient_norm": 0.7881309986114502, "seconds": 0.40322526500006006, "nonpadding_tokens_per_second": 125994.0891848442, "ranks": [{"compute_seconds": 0.39988164199996845, "nonpadding_tokens": 3628, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3974891180000668, "nonpadding_tokens": 16130, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39958489200000713, "nonpadding_tokens": 2826, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.40059762999999293, "nonpadding_tokens": 2993, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.40072866400009843, "nonpadding_tokens": 4337, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39833965000002536, "nonpadding_tokens": 4119, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4002787229999285, "nonpadding_tokens": 8256, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3966777449999199, "nonpadding_tokens": 8515, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003455930000200169, "prefetch": {"data_wait_seconds": 0.009373939001150688, "consumed_batches": 993, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1543, "tokens": 120450624, "loss_tokens": 70816, "loss": 2.4584660864070265, "lr": 0.0002997290402532876, "optimizer_lrs": [0.019981936016885842, 0.0002997290402532876], "gradient_norm": 0.39450424909591675, "seconds": 0.31384963900006824, "nonpadding_tokens_per_second": 225636.7100679845, "ranks": [{"compute_seconds": 0.30600916600008077, "nonpadding_tokens": 7282, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3071274410000342, "nonpadding_tokens": 13255, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3058196439999392, "nonpadding_tokens": 4275, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3107702170000266, "nonpadding_tokens": 20461, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3109622560000389, "nonpadding_tokens": 7930, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30812961800006633, "nonpadding_tokens": 6659, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3104858149999927, "nonpadding_tokens": 4628, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3113961779999954, "nonpadding_tokens": 6326, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004004219999842462, "prefetch": {"data_wait_seconds": 0.009383956001173033, "consumed_batches": 994, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1544, "tokens": 120510164, "loss_tokens": 59540, "loss": 2.5232558993953647, "lr": 0.0002997287190543348, "optimizer_lrs": [0.019981914603622322, 0.0002997287190543348], "gradient_norm": 0.4635794758796692, "seconds": 0.33677125799999885, "nonpadding_tokens_per_second": 176796.5602337721, "ranks": [{"compute_seconds": 0.33396179400006076, "nonpadding_tokens": 10837, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32980077000001984, "nonpadding_tokens": 3870, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32873394900002495, "nonpadding_tokens": 12653, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3298331229999576, "nonpadding_tokens": 2297, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3281973420000668, "nonpadding_tokens": 5995, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3308597689999715, "nonpadding_tokens": 8347, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33320878999995784, "nonpadding_tokens": 4538, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3343146559999468, "nonpadding_tokens": 11003, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003408439999930124, "prefetch": {"data_wait_seconds": 0.009393014001219058, "consumed_batches": 995, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1545, "tokens": 120575048, "loss_tokens": 64884, "loss": 2.4861500427686334, "lr": 0.00029972836880981084, "optimizer_lrs": [0.019981891253987392, 0.00029972836880981084], "gradient_norm": 0.4466923773288727, "seconds": 0.2885283350000236, "nonpadding_tokens_per_second": 224879.1266895665, "ranks": [{"compute_seconds": 0.28612996499998644, "nonpadding_tokens": 7709, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2836940320000849, "nonpadding_tokens": 8451, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2807687929999929, "nonpadding_tokens": 10600, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2811314919999859, "nonpadding_tokens": 11467, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28007662699997127, "nonpadding_tokens": 5285, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.282390362000001, "nonpadding_tokens": 12376, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27949825500002135, "nonpadding_tokens": 4476, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2802828970000064, "nonpadding_tokens": 4520, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038022699993689457, "prefetch": {"data_wait_seconds": 0.009402767001191137, "consumed_batches": 996, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1546, "tokens": 120637015, "loss_tokens": 61967, "loss": 2.737494351832427, "lr": 0.0002997280341005548, "optimizer_lrs": [0.01998186894003699, 0.0002997280341005548], "gradient_norm": 0.4364103376865387, "seconds": 0.33865776400000414, "nonpadding_tokens_per_second": 182978.23521919682, "ranks": [{"compute_seconds": 0.3338608489999615, "nonpadding_tokens": 9809, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33267924099993706, "nonpadding_tokens": 4008, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3351276420000886, "nonpadding_tokens": 9931, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3358454010000287, "nonpadding_tokens": 17273, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33609870500004035, "nonpadding_tokens": 3873, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3334203020000359, "nonpadding_tokens": 7257, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3310376059999953, "nonpadding_tokens": 7378, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3362816879999855, "nonpadding_tokens": 2438, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039425099998879887, "prefetch": {"data_wait_seconds": 0.009411275001255603, "consumed_batches": 997, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1547, "tokens": 120694993, "loss_tokens": 57978, "loss": 2.4149946962641002, "lr": 0.00029972772075109917, "optimizer_lrs": [0.01998184805007328, 0.00029972772075109917], "gradient_norm": 0.39081403613090515, "seconds": 0.4036513759999707, "nonpadding_tokens_per_second": 143633.84704528842, "ranks": [{"compute_seconds": 0.39999515599993174, "nonpadding_tokens": 20218, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3960556089999727, "nonpadding_tokens": 1671, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39725145100010195, "nonpadding_tokens": 1830, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3992131029999655, "nonpadding_tokens": 10227, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3956166469999971, "nonpadding_tokens": 3077, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39685351200000696, "nonpadding_tokens": 10514, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.40105932599999505, "nonpadding_tokens": 4038, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.39624711499993737, "nonpadding_tokens": 6403, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004004209999948216, "prefetch": {"data_wait_seconds": 0.009419072001335849, "consumed_batches": 998, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1548, "tokens": 120752410, "loss_tokens": 57417, "loss": 2.759316165073062, "lr": 0.0002997274102560177, "optimizer_lrs": [0.01998182735040118, 0.0002997274102560177], "gradient_norm": 0.41161009669303894, "seconds": 0.29834146900009273, "nonpadding_tokens_per_second": 192453.96958202333, "ranks": [{"compute_seconds": 0.2951982869999483, "nonpadding_tokens": 9203, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2958534349999127, "nonpadding_tokens": 7431, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29298570700007076, "nonpadding_tokens": 4109, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2937971890000881, "nonpadding_tokens": 2756, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2959795550000308, "nonpadding_tokens": 12864, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29432463900002404, "nonpadding_tokens": 4498, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2959076129999403, "nonpadding_tokens": 4015, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29200597800002015, "nonpadding_tokens": 12541, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037937000001875276, "prefetch": {"data_wait_seconds": 0.009427804001305162, "consumed_batches": 999, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1549, "tokens": 120799446, "loss_tokens": 47036, "loss": 2.2448262700378434, "lr": 0.00029972715576677167, "optimizer_lrs": [0.019981810384451446, 0.00029972715576677167], "gradient_norm": 0.4293070435523987, "seconds": 0.37297805600007905, "nonpadding_tokens_per_second": 126109.29582406861, "ranks": [{"compute_seconds": 0.3691800139999941, "nonpadding_tokens": 11580, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3704478280000103, "nonpadding_tokens": 2843, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3694919670000445, "nonpadding_tokens": 12011, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3701860169999236, "nonpadding_tokens": 2933, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3706424549999383, "nonpadding_tokens": 4285, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36753661900002044, "nonpadding_tokens": 4905, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36652527100000043, "nonpadding_tokens": 2687, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36720535500001006, "nonpadding_tokens": 5792, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000354982999965614, "prefetch": {"data_wait_seconds": 0.009435957001258544, "consumed_batches": 1000, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1550, "tokens": 120865513, "loss_tokens": 66067, "loss": 2.6667998963173747, "lr": 0.00029972679810958834, "optimizer_lrs": [0.019981786540639226, 0.00029972679810958834], "gradient_norm": 0.4266969561576843, "seconds": 0.39246845000002395, "nonpadding_tokens_per_second": 168337.09817947398, "ranks": [{"compute_seconds": 0.38597641899991686, "nonpadding_tokens": 16471, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3858676629999991, "nonpadding_tokens": 8347, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38851611699999467, "nonpadding_tokens": 4291, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.38925749599991377, "nonpadding_tokens": 10717, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3900640290000865, "nonpadding_tokens": 13205, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3865741429999616, "nonpadding_tokens": 7498, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3892985459999636, "nonpadding_tokens": 3094, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3845440889999736, "nonpadding_tokens": 2444, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003706219999912719, "prefetch": {"data_wait_seconds": 0.009445261001317817, "consumed_batches": 1001, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1551, "tokens": 120957646, "loss_tokens": 92133, "loss": 2.6398082459596455, "lr": 0.00029972629895200247, "optimizer_lrs": [0.019981753263466833, 0.00029972629895200247], "gradient_norm": 0.41653215885162354, "seconds": 0.2871212080000305, "nonpadding_tokens_per_second": 320885.38719156623, "ranks": [{"compute_seconds": 0.2831720909999831, "nonpadding_tokens": 7878, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28152927099995395, "nonpadding_tokens": 24448, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2819433490000165, "nonpadding_tokens": 3898, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2845786350000026, "nonpadding_tokens": 17778, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2809803310000234, "nonpadding_tokens": 23806, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2827858910000032, "nonpadding_tokens": 5479, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28478060999998434, "nonpadding_tokens": 6630, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2821328480000602, "nonpadding_tokens": 2216, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044569399994998093, "prefetch": {"data_wait_seconds": 0.009453488001284427, "consumed_batches": 1002, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1552, "tokens": 121029779, "loss_tokens": 72133, "loss": 2.6575568307848, "lr": 0.0002997259078326431, "optimizer_lrs": [0.019981727188842873, 0.0002997259078326431], "gradient_norm": 0.4223456084728241, "seconds": 0.2669620469999927, "nonpadding_tokens_per_second": 270199.4564793024, "ranks": [{"compute_seconds": 0.26469601999997394, "nonpadding_tokens": 4804, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2642803680000725, "nonpadding_tokens": 6841, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26119532400002754, "nonpadding_tokens": 7162, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2620156760000327, "nonpadding_tokens": 8159, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26011361899998064, "nonpadding_tokens": 3950, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26290297200000623, "nonpadding_tokens": 28485, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26399482900001203, "nonpadding_tokens": 2734, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26092362800000046, "nonpadding_tokens": 9998, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037248000001000037, "prefetch": {"data_wait_seconds": 0.009462445001261699, "consumed_batches": 1003, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1553, "tokens": 121089519, "loss_tokens": 59740, "loss": 2.3686037935219284, "lr": 0.0002997255836993802, "optimizer_lrs": [0.01998170557995868, 0.0002997255836993802], "gradient_norm": 0.37556084990501404, "seconds": 0.3395369419999952, "nonpadding_tokens_per_second": 175945.50875115336, "ranks": [{"compute_seconds": 0.33569632499995805, "nonpadding_tokens": 1119, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3363694199999827, "nonpadding_tokens": 17601, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3358975459999556, "nonpadding_tokens": 5425, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3341668650000429, "nonpadding_tokens": 7717, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33256036600005245, "nonpadding_tokens": 6993, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3359024579999641, "nonpadding_tokens": 3566, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3324880369999619, "nonpadding_tokens": 3976, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33721792500000447, "nonpadding_tokens": 13343, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003486020000309509, "prefetch": {"data_wait_seconds": 0.009471506001275998, "consumed_batches": 1004, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1554, "tokens": 121173819, "loss_tokens": 84300, "loss": 2.520698769276394, "lr": 0.00029972512598461026, "optimizer_lrs": [0.019981675065640685, 0.00029972512598461026], "gradient_norm": 0.35858622193336487, "seconds": 0.2983750209999698, "nonpadding_tokens_per_second": 282530.35296814784, "ranks": [{"compute_seconds": 0.29387438000003385, "nonpadding_tokens": 7927, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29573207299995374, "nonpadding_tokens": 13804, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29268120100005035, "nonpadding_tokens": 13290, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2960506419999547, "nonpadding_tokens": 4878, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2958345790000294, "nonpadding_tokens": 18091, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2929965359999187, "nonpadding_tokens": 15233, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29135734799990587, "nonpadding_tokens": 7971, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2921956870000031, "nonpadding_tokens": 3106, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037134800004423596, "prefetch": {"data_wait_seconds": 0.009479962001208264, "consumed_batches": 1005, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1555, "tokens": 121238677, "loss_tokens": 64858, "loss": 2.359106384331925, "lr": 0.00029972477357266563, "optimizer_lrs": [0.019981651571511043, 0.00029972477357266563], "gradient_norm": 0.40103277564048767, "seconds": 0.3791005389999782, "nonpadding_tokens_per_second": 171083.90341804217, "ranks": [{"compute_seconds": 0.3768079309999166, "nonpadding_tokens": 6353, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37309702100003506, "nonpadding_tokens": 8109, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3757990310000423, "nonpadding_tokens": 12119, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.37622270800000024, "nonpadding_tokens": 4429, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.37232173400002466, "nonpadding_tokens": 4804, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37539425799991477, "nonpadding_tokens": 10701, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3724460820000104, "nonpadding_tokens": 6717, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37337199099999907, "nonpadding_tokens": 11626, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003756600000315302, "prefetch": {"data_wait_seconds": 0.009488781001167013, "consumed_batches": 1006, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1556, "tokens": 121303394, "loss_tokens": 64717, "loss": 2.4631805592039187, "lr": 0.00029972442170205665, "optimizer_lrs": [0.019981628113470446, 0.00029972442170205665], "gradient_norm": 0.3857616186141968, "seconds": 0.3268713249999564, "nonpadding_tokens_per_second": 197989.22404713422, "ranks": [{"compute_seconds": 0.304094389999932, "nonpadding_tokens": 17031, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3112484759999461, "nonpadding_tokens": 3070, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31021801999997933, "nonpadding_tokens": 11145, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30895632299996123, "nonpadding_tokens": 8193, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31124291300000095, "nonpadding_tokens": 2840, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.309767344000079, "nonpadding_tokens": 1967, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30677445000003445, "nonpadding_tokens": 11531, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31176921500002663, "nonpadding_tokens": 8940, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.002742672000067614, "prefetch": {"data_wait_seconds": 0.009497546001128399, "consumed_batches": 1007, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1557, "tokens": 121381204, "loss_tokens": 77810, "loss": 2.255434712761856, "lr": 0.00029972399834664424, "optimizer_lrs": [0.019981599889776283, 0.00029972399834664424], "gradient_norm": 0.4177311658859253, "seconds": 0.2919320810000272, "nonpadding_tokens_per_second": 266534.5985047555, "ranks": [{"compute_seconds": 0.274023102000001, "nonpadding_tokens": 10423, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28398536999998214, "nonpadding_tokens": 5387, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2847039399999858, "nonpadding_tokens": 9772, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2850696149999976, "nonpadding_tokens": 4957, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2836520580000297, "nonpadding_tokens": 5994, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2848971810000194, "nonpadding_tokens": 13968, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2876396300000579, "nonpadding_tokens": 3363, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28409710500000074, "nonpadding_tokens": 23946, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003951439999809736, "prefetch": {"data_wait_seconds": 0.009506764001073975, "consumed_batches": 1008, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1558, "tokens": 121458428, "loss_tokens": 77224, "loss": 2.824975477183259, "lr": 0.0002997235778586431, "optimizer_lrs": [0.019981571857242874, 0.0002997235778586431], "gradient_norm": 0.40780550241470337, "seconds": 0.22394540200002666, "nonpadding_tokens_per_second": 344834.05022082484, "ranks": [{"compute_seconds": 0.22144677899996168, "nonpadding_tokens": 4154, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21778882899991459, "nonpadding_tokens": 24610, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21806994899998244, "nonpadding_tokens": 12212, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22095124700001634, "nonpadding_tokens": 11839, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21700552000004336, "nonpadding_tokens": 8168, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22014788500007398, "nonpadding_tokens": 7322, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21709093200001917, "nonpadding_tokens": 6305, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22164218400007485, "nonpadding_tokens": 2614, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004182569999784391, "prefetch": {"data_wait_seconds": 0.009516412001019603, "consumed_batches": 1009, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1559, "tokens": 121524131, "loss_tokens": 65703, "loss": 2.3715457722630626, "lr": 0.0002997232198512413, "optimizer_lrs": [0.019981547990082754, 0.0002997232198512413], "gradient_norm": 0.4307486116886139, "seconds": 0.2546174130000054, "nonpadding_tokens_per_second": 258045.98053943235, "ranks": [{"compute_seconds": 0.2499947889999703, "nonpadding_tokens": 6631, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2519752770000423, "nonpadding_tokens": 7312, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.248773730000039, "nonpadding_tokens": 16222, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2499220920000198, "nonpadding_tokens": 11072, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25211641800001416, "nonpadding_tokens": 8268, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25046857200004524, "nonpadding_tokens": 9080, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24758308899993153, "nonpadding_tokens": 3064, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25225002199999835, "nonpadding_tokens": 4054, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036410799998520815, "prefetch": {"data_wait_seconds": 0.00952657100101817, "consumed_batches": 1010, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1560, "tokens": 121591624, "loss_tokens": 67493, "loss": 2.4711426092335502, "lr": 0.00029972285184936165, "optimizer_lrs": [0.01998152345662411, 0.00029972285184936165], "gradient_norm": 0.4989334046840668, "seconds": 0.3298975129999917, "nonpadding_tokens_per_second": 204587.7805692997, "ranks": [{"compute_seconds": 0.29576649900002394, "nonpadding_tokens": 8278, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31296026000006805, "nonpadding_tokens": 25532, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3118503089999649, "nonpadding_tokens": 3627, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3120360799999844, "nonpadding_tokens": 11968, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30857861599997705, "nonpadding_tokens": 4949, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3098951089999673, "nonpadding_tokens": 3502, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31220950700003414, "nonpadding_tokens": 3422, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3091608280000173, "nonpadding_tokens": 6215, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004364579999673879, "prefetch": {"data_wait_seconds": 0.009535115000929295, "consumed_batches": 1011, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1561, "tokens": 121641664, "loss_tokens": 50040, "loss": 2.6864033772981615, "lr": 0.00029972257885134407, "optimizer_lrs": [0.019981505256756273, 0.00029972257885134407], "gradient_norm": 0.4488564729690552, "seconds": 0.2860405899999705, "nonpadding_tokens_per_second": 174940.20691261042, "ranks": [{"compute_seconds": 0.28200600900004247, "nonpadding_tokens": 6431, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27929515300002095, "nonpadding_tokens": 4290, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27992616200003795, "nonpadding_tokens": 7186, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.282487739999965, "nonpadding_tokens": 13086, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2832191759999887, "nonpadding_tokens": 7299, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2803609280000501, "nonpadding_tokens": 3624, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28352752600005715, "nonpadding_tokens": 5225, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27957359200001974, "nonpadding_tokens": 2899, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039700500008166273, "prefetch": {"data_wait_seconds": 0.009543761000898598, "consumed_batches": 1012, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1562, "tokens": 121680082, "loss_tokens": 38418, "loss": 2.4876597965146545, "lr": 0.00029972236916716075, "optimizer_lrs": [0.01998149127781072, 0.00029972236916716075], "gradient_norm": 0.433386892080307, "seconds": 0.2692093709999881, "nonpadding_tokens_per_second": 142706.77078325665, "ranks": [{"compute_seconds": 0.2667959000000337, "nonpadding_tokens": 4553, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26276609199999257, "nonpadding_tokens": 3896, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26117060200010656, "nonpadding_tokens": 3588, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26389633600001616, "nonpadding_tokens": 2361, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26472120099992935, "nonpadding_tokens": 2482, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2618194220000305, "nonpadding_tokens": 5039, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2586644630000592, "nonpadding_tokens": 14116, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2595182839999097, "nonpadding_tokens": 2383, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036129099999016034, "prefetch": {"data_wait_seconds": 0.00955247900090228, "consumed_batches": 1013, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1563, "tokens": 121744670, "loss_tokens": 64588, "loss": 2.804410503886171, "lr": 0.0002997220164696294, "optimizer_lrs": [0.019981467764641964, 0.0002997220164696294], "gradient_norm": 0.6347411870956421, "seconds": 0.2655692150000277, "nonpadding_tokens_per_second": 243205.90020192385, "ranks": [{"compute_seconds": 0.26299309500006984, "nonpadding_tokens": 1644, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26240564099998664, "nonpadding_tokens": 7381, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2615231480000375, "nonpadding_tokens": 21202, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26216245699993124, "nonpadding_tokens": 1796, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26297228199996425, "nonpadding_tokens": 5469, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2598136719999502, "nonpadding_tokens": 7417, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2625505339999563, "nonpadding_tokens": 16146, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2591056479999452, "nonpadding_tokens": 3533, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039789499999187683, "prefetch": {"data_wait_seconds": 0.00956193300096686, "consumed_batches": 1014, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1564, "tokens": 121852531, "loss_tokens": 107861, "loss": 2.6971550769045347, "lr": 0.00029972142697136893, "optimizer_lrs": [0.01998142846475793, 0.00029972142697136893], "gradient_norm": 0.35967278480529785, "seconds": 0.33173594899994896, "nonpadding_tokens_per_second": 325141.12602254207, "ranks": [{"compute_seconds": 0.32924664299991946, "nonpadding_tokens": 16878, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3294647469999745, "nonpadding_tokens": 5181, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3267347419999851, "nonpadding_tokens": 15927, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.328950682000027, "nonpadding_tokens": 5049, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32936561599990455, "nonpadding_tokens": 18943, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3262628370000584, "nonpadding_tokens": 7927, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3255841060000648, "nonpadding_tokens": 5188, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32628625600000305, "nonpadding_tokens": 32768, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004318609999245382, "prefetch": {"data_wait_seconds": 0.009572654001047454, "consumed_batches": 1015, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1565, "tokens": 121936133, "loss_tokens": 83602, "loss": 2.2748220362551135, "lr": 0.00029972096962797547, "optimizer_lrs": [0.019981397975198364, 0.00029972096962797547], "gradient_norm": 0.4074608385562897, "seconds": 0.280510554999978, "nonpadding_tokens_per_second": 298035.13097753684, "ranks": [{"compute_seconds": 0.2781788280000228, "nonpadding_tokens": 5180, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2748889040000222, "nonpadding_tokens": 6197, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2737991079999347, "nonpadding_tokens": 20987, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2745781060000354, "nonpadding_tokens": 11880, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27261908800005585, "nonpadding_tokens": 11766, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.275901355999963, "nonpadding_tokens": 10235, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2728077240000175, "nonpadding_tokens": 4963, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27362917799996467, "nonpadding_tokens": 12394, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003980550000051153, "prefetch": {"data_wait_seconds": 0.009581566001045871, "consumed_batches": 1016, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1566, "tokens": 122004086, "loss_tokens": 67953, "loss": 2.56095642392536, "lr": 0.00029972059761611047, "optimizer_lrs": [0.019981373174407367, 0.00029972059761611047], "gradient_norm": 0.35294798016548157, "seconds": 0.31469512900002883, "nonpadding_tokens_per_second": 215932.79888356256, "ranks": [{"compute_seconds": 0.24390984600006504, "nonpadding_tokens": 4544, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25830463600004805, "nonpadding_tokens": 3308, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25720011600003545, "nonpadding_tokens": 7674, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2555183089999673, "nonpadding_tokens": 5450, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2582868880000433, "nonpadding_tokens": 12157, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2573882039999944, "nonpadding_tokens": 16908, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2582568640000318, "nonpadding_tokens": 12266, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2585863199999494, "nonpadding_tokens": 5646, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.021131007999997564, "prefetch": {"data_wait_seconds": 0.009590256001118291, "consumed_batches": 1017, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1567, "tokens": 122075484, "loss_tokens": 71398, "loss": 2.3463776558867195, "lr": 0.00029972020647772127, "optimizer_lrs": [0.019981347098514752, 0.00029972020647772127], "gradient_norm": 0.3565208315849304, "seconds": 0.30550059500001225, "nonpadding_tokens_per_second": 233708.21912800902, "ranks": [{"compute_seconds": 0.2826208640000232, "nonpadding_tokens": 16020, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.282901207000009, "nonpadding_tokens": 2076, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2840120249999245, "nonpadding_tokens": 12569, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28059440700008054, "nonpadding_tokens": 11977, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2847642070000802, "nonpadding_tokens": 4985, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2819121530000075, "nonpadding_tokens": 9646, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28433788500001356, "nonpadding_tokens": 6559, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2799019470000985, "nonpadding_tokens": 7566, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0043157269999483105, "prefetch": {"data_wait_seconds": 0.00959781600109011, "consumed_batches": 1018, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1568, "tokens": 122134242, "loss_tokens": 58758, "loss": 2.5010229989958814, "lr": 0.00029971988437980853, "optimizer_lrs": [0.01998132562532057, 0.00029971988437980853], "gradient_norm": 0.409291535615921, "seconds": 0.2634156780000012, "nonpadding_tokens_per_second": 223061.8938330608, "ranks": [{"compute_seconds": 0.26080426099997567, "nonpadding_tokens": 2288, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26120340600004965, "nonpadding_tokens": 5958, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2579531249999718, "nonpadding_tokens": 9304, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2607714289999876, "nonpadding_tokens": 5123, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26110503899997184, "nonpadding_tokens": 3885, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25840166700004374, "nonpadding_tokens": 2721, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2608046289999493, "nonpadding_tokens": 13955, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2575469419999763, "nonpadding_tokens": 15524, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000337944000079915, "prefetch": {"data_wait_seconds": 0.00960645500113344, "consumed_batches": 1019, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1569, "tokens": 122190176, "loss_tokens": 55934, "loss": 2.6114175859048165, "lr": 0.00029971957759044796, "optimizer_lrs": [0.019981305172696532, 0.00029971957759044796], "gradient_norm": 0.9235744476318359, "seconds": 0.2791956390000223, "nonpadding_tokens_per_second": 200339.80545088503, "ranks": [{"compute_seconds": 0.23322466499996608, "nonpadding_tokens": 3628, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2565179519999674, "nonpadding_tokens": 1658, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2584644710000248, "nonpadding_tokens": 1222, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2596597390000852, "nonpadding_tokens": 6538, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2600526630000104, "nonpadding_tokens": 4115, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25584925099997236, "nonpadding_tokens": 2806, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2530409170000212, "nonpadding_tokens": 11028, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2604548500000874, "nonpadding_tokens": 24939, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0006943810000166195, "prefetch": {"data_wait_seconds": 0.009615346001126568, "consumed_batches": 1020, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1570, "tokens": 122299762, "loss_tokens": 109586, "loss": 2.721225510101655, "lr": 0.000299718976041909, "optimizer_lrs": [0.019981265069460602, 0.000299718976041909], "gradient_norm": 0.4285254180431366, "seconds": 0.41843185400000493, "nonpadding_tokens_per_second": 261896.88703766494, "ranks": [{"compute_seconds": 0.41559369200001584, "nonpadding_tokens": 10979, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4161315220000006, "nonpadding_tokens": 9396, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4134925399999929, "nonpadding_tokens": 22588, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.413334903999953, "nonpadding_tokens": 11860, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4120654519999789, "nonpadding_tokens": 28135, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41350063100003354, "nonpadding_tokens": 4432, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4158990570000469, "nonpadding_tokens": 17958, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4127484690000074, "nonpadding_tokens": 4238, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040040799990492815, "prefetch": {"data_wait_seconds": 0.009624374001148794, "consumed_batches": 1021, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1571, "tokens": 122368814, "loss_tokens": 69052, "loss": 2.6560850427938365, "lr": 0.0002997185966652903, "optimizer_lrs": [0.01998123977768602, 0.0002997185966652903], "gradient_norm": 0.46401622891426086, "seconds": 0.2899537660000533, "nonpadding_tokens_per_second": 238148.3122381218, "ranks": [{"compute_seconds": 0.2868706019999081, "nonpadding_tokens": 2906, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28416807799999333, "nonpadding_tokens": 3227, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2844074060000139, "nonpadding_tokens": 11717, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28505837299996983, "nonpadding_tokens": 20445, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2876420140000846, "nonpadding_tokens": 8233, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28632804900007613, "nonpadding_tokens": 4451, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2834860400000707, "nonpadding_tokens": 11052, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28397978700002113, "nonpadding_tokens": 7021, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039705400001821545, "prefetch": {"data_wait_seconds": 0.009633204001218587, "consumed_batches": 1022, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1572, "tokens": 122429388, "loss_tokens": 60574, "loss": 2.4580624009476013, "lr": 0.00029971826365692597, "optimizer_lrs": [0.0199812175771284, 0.00029971826365692597], "gradient_norm": 0.5373871922492981, "seconds": 0.3209888829999272, "nonpadding_tokens_per_second": 188710.58534452028, "ranks": [{"compute_seconds": 0.3161730250000119, "nonpadding_tokens": 6524, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3171682940001119, "nonpadding_tokens": 9966, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31347138699993593, "nonpadding_tokens": 4026, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3173293699999249, "nonpadding_tokens": 15751, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3184572099999059, "nonpadding_tokens": 10375, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31478947499999776, "nonpadding_tokens": 8155, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3174734869999156, "nonpadding_tokens": 3180, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31398615599994173, "nonpadding_tokens": 2597, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035059300000739313, "prefetch": {"data_wait_seconds": 0.009642406001219683, "consumed_batches": 1023, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1573, "tokens": 122481651, "loss_tokens": 52263, "loss": 2.716522982320188, "lr": 0.00029971797618058587, "optimizer_lrs": [0.01998119841203906, 0.00029971797618058587], "gradient_norm": 0.4348266124725342, "seconds": 0.2817642859999978, "nonpadding_tokens_per_second": 185484.82755547098, "ranks": [{"compute_seconds": 0.2774791250000135, "nonpadding_tokens": 17657, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27432637999993403, "nonpadding_tokens": 4342, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27746847900004923, "nonpadding_tokens": 4065, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2717050779999681, "nonpadding_tokens": 5320, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27068719799990504, "nonpadding_tokens": 4646, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2757904979999921, "nonpadding_tokens": 5154, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27040974900000947, "nonpadding_tokens": 6993, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27929144800009453, "nonpadding_tokens": 4086, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035982599990802555, "prefetch": {"data_wait_seconds": 0.009651357001146721, "consumed_batches": 1024, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1574, "tokens": 122594473, "loss_tokens": 112822, "loss": 2.36746406950772, "lr": 0.00029971735509589644, "optimizer_lrs": [0.019981157006393097, 0.00029971735509589644], "gradient_norm": 0.6383209824562073, "seconds": 0.3663664329999392, "nonpadding_tokens_per_second": 307948.5177617752, "ranks": [{"compute_seconds": 0.3638668590000407, "nonpadding_tokens": 20148, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36071840700003577, "nonpadding_tokens": 10802, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3579452469999751, "nonpadding_tokens": 24283, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3629299079999555, "nonpadding_tokens": 9878, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3632542049999756, "nonpadding_tokens": 27419, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36034914700007903, "nonpadding_tokens": 7472, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3571700240000837, "nonpadding_tokens": 7071, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3572111359999326, "nonpadding_tokens": 5749, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004425370000262774, "prefetch": {"data_wait_seconds": 0.009677924001110227, "consumed_batches": 1025, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1575, "tokens": 122672607, "loss_tokens": 78134, "loss": 2.692317045076407, "lr": 0.0002997169245685654, "optimizer_lrs": [0.01998112830457103, 0.0002997169245685654], "gradient_norm": 0.3884192705154419, "seconds": 0.2518420200000264, "nonpadding_tokens_per_second": 310250.0527909989, "ranks": [{"compute_seconds": 0.24910210899997764, "nonpadding_tokens": 4720, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24576010900000256, "nonpadding_tokens": 12698, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24847817199997735, "nonpadding_tokens": 6022, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24683095099999264, "nonpadding_tokens": 9430, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.249423041, "nonpadding_tokens": 10251, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24682041299990942, "nonpadding_tokens": 4166, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2451738940000041, "nonpadding_tokens": 19808, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2461586739999575, "nonpadding_tokens": 11039, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003853599999956714, "prefetch": {"data_wait_seconds": 0.00968781700107968, "consumed_batches": 1026, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1576, "tokens": 122746668, "loss_tokens": 74061, "loss": 2.3390039123155235, "lr": 0.00029971651618178265, "optimizer_lrs": [0.01998110107878551, 0.00029971651618178265], "gradient_norm": 0.4372968077659607, "seconds": 0.2533627500000648, "nonpadding_tokens_per_second": 292312.1098108584, "ranks": [{"compute_seconds": 0.2482513119999794, "nonpadding_tokens": 3726, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2509219699999221, "nonpadding_tokens": 3083, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24811184899999716, "nonpadding_tokens": 19198, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24859455900002558, "nonpadding_tokens": 4186, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24917142000003878, "nonpadding_tokens": 9789, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2496853850000207, "nonpadding_tokens": 13420, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2469936490000464, "nonpadding_tokens": 4963, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24741012699996645, "nonpadding_tokens": 15696, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005193980000512965, "prefetch": {"data_wait_seconds": 0.009694872001048793, "consumed_batches": 1027, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1577, "tokens": 122795754, "loss_tokens": 49086, "loss": 2.6545433333842645, "lr": 0.00029971624534997883, "optimizer_lrs": [0.019981083023331923, 0.00029971624534997883], "gradient_norm": 0.43296679854393005, "seconds": 0.3276120240000182, "nonpadding_tokens_per_second": 149829.6655924853, "ranks": [{"compute_seconds": 0.3248854500000107, "nonpadding_tokens": 4370, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3216987140000356, "nonpadding_tokens": 2732, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3242762419999963, "nonpadding_tokens": 13644, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3227439400000094, "nonpadding_tokens": 2782, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32523310400006267, "nonpadding_tokens": 1692, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3238218539999025, "nonpadding_tokens": 10962, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3210092799999984, "nonpadding_tokens": 3887, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3215138449999131, "nonpadding_tokens": 9017, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003429820000064865, "prefetch": {"data_wait_seconds": 0.009702914001039176, "consumed_batches": 1028, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1578, "tokens": 122857735, "loss_tokens": 61981, "loss": 2.646367183088366, "lr": 0.00029971590318553215, "optimizer_lrs": [0.019981060212368812, 0.00029971590318553215], "gradient_norm": 0.4067094624042511, "seconds": 0.4186343380000608, "nonpadding_tokens_per_second": 148055.22235968852, "ranks": [{"compute_seconds": 0.39094169099996634, "nonpadding_tokens": 6794, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3944862259999127, "nonpadding_tokens": 6274, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3938409180000235, "nonpadding_tokens": 9592, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.391991839999946, "nonpadding_tokens": 9907, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3906777280000142, "nonpadding_tokens": 8199, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3932641029999786, "nonpadding_tokens": 9269, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.39455094499999177, "nonpadding_tokens": 5198, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.39133372599997074, "nonpadding_tokens": 6748, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.005680773999984012, "prefetch": {"data_wait_seconds": 0.009711770001061382, "consumed_batches": 1029, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1579, "tokens": 122902977, "loss_tokens": 45242, "loss": 2.469731527120817, "lr": 0.0002997156532982507, "optimizer_lrs": [0.019981043553216716, 0.0002997156532982507], "gradient_norm": 0.43596020340919495, "seconds": 0.28842223200001627, "nonpadding_tokens_per_second": 156860.30749528852, "ranks": [{"compute_seconds": 0.2236199779999879, "nonpadding_tokens": 3705, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23649336799996945, "nonpadding_tokens": 11467, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2341242740000098, "nonpadding_tokens": 9756, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23413546100005078, "nonpadding_tokens": 3745, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2327113719999261, "nonpadding_tokens": 1646, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23441470399995978, "nonpadding_tokens": 2714, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2326112869999406, "nonpadding_tokens": 6941, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2374092649999966, "nonpadding_tokens": 5268, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.03430233799997495, "prefetch": {"data_wait_seconds": 0.00972159500106784, "consumed_batches": 1030, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1580, "tokens": 122954869, "loss_tokens": 51892, "loss": 2.6769576717027674, "lr": 0.0002997153665455953, "optimizer_lrs": [0.01998102443637302, 0.0002997153665455953], "gradient_norm": 0.40525349974632263, "seconds": 0.2577249619999975, "nonpadding_tokens_per_second": 201346.42604002208, "ranks": [{"compute_seconds": 0.2552090420000468, "nonpadding_tokens": 3867, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2540911580000511, "nonpadding_tokens": 7902, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25390627799993126, "nonpadding_tokens": 4858, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25453502300001674, "nonpadding_tokens": 9492, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2504454989999658, "nonpadding_tokens": 15693, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25194605900003353, "nonpadding_tokens": 3875, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2504083310000169, "nonpadding_tokens": 3317, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2553297860000612, "nonpadding_tokens": 2888, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003564950000054523, "prefetch": {"data_wait_seconds": 0.009730590001097994, "consumed_batches": 1031, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1581, "tokens": 123033295, "loss_tokens": 78426, "loss": 2.5889931272792186, "lr": 0.0002997149328933894, "optimizer_lrs": [0.019980995526225964, 0.0002997149328933894], "gradient_norm": 0.4066137373447418, "seconds": 0.31874102900007983, "nonpadding_tokens_per_second": 246049.277829182, "ranks": [{"compute_seconds": 0.31484192699997493, "nonpadding_tokens": 9767, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31261629699997684, "nonpadding_tokens": 10048, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31284056000004057, "nonpadding_tokens": 5838, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3153147819999731, "nonpadding_tokens": 9936, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3113345599999775, "nonpadding_tokens": 13926, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3131692590000057, "nonpadding_tokens": 4471, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31558221700004196, "nonpadding_tokens": 12440, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31622439399995983, "nonpadding_tokens": 12000, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005200939999667753, "prefetch": {"data_wait_seconds": 0.009737187001135794, "consumed_batches": 1032, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1582, "tokens": 123099085, "loss_tokens": 65790, "loss": 2.859328687870497, "lr": 0.00029971456885691136, "optimizer_lrs": [0.019980971257127427, 0.00029971456885691136], "gradient_norm": 0.3853060305118561, "seconds": 0.23079032800001187, "nonpadding_tokens_per_second": 285063.93907458987, "ranks": [{"compute_seconds": 0.18804277399999592, "nonpadding_tokens": 4139, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.1881706450000138, "nonpadding_tokens": 8283, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19116483900006642, "nonpadding_tokens": 5821, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.18967894299998989, "nonpadding_tokens": 10677, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.19163712299996405, "nonpadding_tokens": 9271, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19094641500009857, "nonpadding_tokens": 4899, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.18756529000006594, "nonpadding_tokens": 10619, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.18847039900003892, "nonpadding_tokens": 12081, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.027312376000054428, "prefetch": {"data_wait_seconds": 0.009745299001181706, "consumed_batches": 1033, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1583, "tokens": 123178060, "loss_tokens": 78975, "loss": 2.377650759734093, "lr": 0.0002997141315572748, "optimizer_lrs": [0.019980942103818324, 0.0002997141315572748], "gradient_norm": 0.5041322112083435, "seconds": 0.34089206999999533, "nonpadding_tokens_per_second": 231671.56689799525, "ranks": [{"compute_seconds": 0.33726565100005246, "nonpadding_tokens": 10603, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33809899100003804, "nonpadding_tokens": 15750, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3376196130000153, "nonpadding_tokens": 10974, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3376903859999629, "nonpadding_tokens": 11979, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3346208519999436, "nonpadding_tokens": 10662, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3354342269999506, "nonpadding_tokens": 2270, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33414659800007485, "nonpadding_tokens": 3747, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3386069780000298, "nonpadding_tokens": 12990, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033675199995286675, "prefetch": {"data_wait_seconds": 0.00975326900113771, "consumed_batches": 1034, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1584, "tokens": 123239246, "loss_tokens": 61186, "loss": 2.517828067286634, "lr": 0.0002997137925288426, "optimizer_lrs": [0.01998091950192284, 0.0002997137925288426], "gradient_norm": 0.40330973267555237, "seconds": 0.2485768999999891, "nonpadding_tokens_per_second": 246145.15668995262, "ranks": [{"compute_seconds": 0.24562208099996496, "nonpadding_tokens": 7264, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24593894899999214, "nonpadding_tokens": 20044, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24515567100002045, "nonpadding_tokens": 12101, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24399198499997965, "nonpadding_tokens": 10619, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2462801040001068, "nonpadding_tokens": 5101, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24475478699991982, "nonpadding_tokens": 2114, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24606204599990633, "nonpadding_tokens": 2355, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2425950729999613, "nonpadding_tokens": 1588, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003601050000270334, "prefetch": {"data_wait_seconds": 0.009762145001175782, "consumed_batches": 1035, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1585, "tokens": 123295234, "loss_tokens": 55988, "loss": 2.6152247468207475, "lr": 0.0002997134821264042, "optimizer_lrs": [0.019980898808426947, 0.0002997134821264042], "gradient_norm": 0.4370349645614624, "seconds": 0.29846406199999365, "nonpadding_tokens_per_second": 187587.07371610185, "ranks": [{"compute_seconds": 0.2961985619999723, "nonpadding_tokens": 5807, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29215433299998494, "nonpadding_tokens": 12172, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2948905679999143, "nonpadding_tokens": 10279, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29592381700001624, "nonpadding_tokens": 3953, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2957263910000165, "nonpadding_tokens": 3723, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29303613599995515, "nonpadding_tokens": 6743, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2918718550000676, "nonpadding_tokens": 5069, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2918780479999441, "nonpadding_tokens": 8242, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041157500004374015, "prefetch": {"data_wait_seconds": 0.009790064001165888, "consumed_batches": 1036, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1586, "tokens": 123343978, "loss_tokens": 48744, "loss": 2.5215249248625473, "lr": 0.00029971321174850134, "optimizer_lrs": [0.019980880783233426, 0.00029971321174850134], "gradient_norm": 0.40409183502197266, "seconds": 0.2979425100000981, "nonpadding_tokens_per_second": 163602.03181474155, "ranks": [{"compute_seconds": 0.2956019149999065, "nonpadding_tokens": 1459, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2921003840000367, "nonpadding_tokens": 19436, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2950078339999891, "nonpadding_tokens": 6746, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2952686569999514, "nonpadding_tokens": 3242, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2917567730000883, "nonpadding_tokens": 2219, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2928506890000335, "nonpadding_tokens": 3512, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2956262540000125, "nonpadding_tokens": 3924, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2919254700000238, "nonpadding_tokens": 8206, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034514699996179843, "prefetch": {"data_wait_seconds": 0.009798031001153618, "consumed_batches": 1037, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1587, "tokens": 123424017, "loss_tokens": 80039, "loss": 2.554756704544035, "lr": 0.0002997127675041852, "optimizer_lrs": [0.01998085116694568, 0.0002997127675041852], "gradient_norm": 0.46789491176605225, "seconds": 0.3372435310000128, "nonpadding_tokens_per_second": 237332.9438304243, "ranks": [{"compute_seconds": 0.3347106659999781, "nonpadding_tokens": 17567, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32993977399996766, "nonpadding_tokens": 12668, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33331962700003714, "nonpadding_tokens": 5875, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33179511100001946, "nonpadding_tokens": 12084, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3301063060000615, "nonpadding_tokens": 12938, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33384392699997534, "nonpadding_tokens": 9307, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3318267019999439, "nonpadding_tokens": 3828, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3339747539999962, "nonpadding_tokens": 5772, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038776799999595823, "prefetch": {"data_wait_seconds": 0.009807382001056908, "consumed_batches": 1038, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1588, "tokens": 123479961, "loss_tokens": 55944, "loss": 2.832328980766481, "lr": 0.00029971245679158876, "optimizer_lrs": [0.019980830452772585, 0.00029971245679158876], "gradient_norm": 1.0198537111282349, "seconds": 0.28711596200002987, "nonpadding_tokens_per_second": 194848.1011306302, "ranks": [{"compute_seconds": 0.28445593399999325, "nonpadding_tokens": 11648, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2841633489999822, "nonpadding_tokens": 4779, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28124026900002264, "nonpadding_tokens": 8275, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.282182081999963, "nonpadding_tokens": 3587, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28480509799999254, "nonpadding_tokens": 2481, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28187899499994273, "nonpadding_tokens": 2394, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28466900900002656, "nonpadding_tokens": 10891, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2813785140000391, "nonpadding_tokens": 11889, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003606450000006589, "prefetch": {"data_wait_seconds": 0.00981596700098919, "consumed_batches": 1039, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1589, "tokens": 123562932, "loss_tokens": 82971, "loss": 2.6803913415530727, "lr": 0.00029971199566226344, "optimizer_lrs": [0.019980799710817565, 0.00029971199566226344], "gradient_norm": 0.4183267056941986, "seconds": 0.2705210909999778, "nonpadding_tokens_per_second": 306708.0636607621, "ranks": [{"compute_seconds": 0.2645597140000291, "nonpadding_tokens": 10000, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26402974000006907, "nonpadding_tokens": 6607, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26663818099996206, "nonpadding_tokens": 19228, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26734180500000093, "nonpadding_tokens": 12477, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26124507500003347, "nonpadding_tokens": 12672, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2650098110000272, "nonpadding_tokens": 14515, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26799222799991185, "nonpadding_tokens": 2306, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2621233849999953, "nonpadding_tokens": 5166, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040228700004263374, "prefetch": {"data_wait_seconds": 0.009830717000980371, "consumed_batches": 1040, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1590, "tokens": 123656564, "loss_tokens": 93632, "loss": 2.494492734855605, "lr": 0.00029971147483887794, "optimizer_lrs": [0.01998076498925853, 0.00029971147483887794], "gradient_norm": 0.4084484279155731, "seconds": 0.3812258650000331, "nonpadding_tokens_per_second": 245607.67932152734, "ranks": [{"compute_seconds": 0.3763525299999628, "nonpadding_tokens": 11435, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37867768700004945, "nonpadding_tokens": 15170, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37814312800003336, "nonpadding_tokens": 8019, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3769975950000344, "nonpadding_tokens": 9555, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3749356399999897, "nonpadding_tokens": 19378, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3762333920000174, "nonpadding_tokens": 11050, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3788295639999433, "nonpadding_tokens": 5954, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3769455569999991, "nonpadding_tokens": 13071, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003941900000654641, "prefetch": {"data_wait_seconds": 0.009839140001034252, "consumed_batches": 1041, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1591, "tokens": 123712817, "loss_tokens": 56253, "loss": 2.5283001573249426, "lr": 0.0002997111617083319, "optimizer_lrs": [0.019980744113888796, 0.0002997111617083319], "gradient_norm": 0.37494173645973206, "seconds": 0.23143057699996916, "nonpadding_tokens_per_second": 243066.41209302042, "ranks": [{"compute_seconds": 0.2039797229999749, "nonpadding_tokens": 4571, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20468294299996614, "nonpadding_tokens": 3529, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20535131999997702, "nonpadding_tokens": 9556, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.20798986499994498, "nonpadding_tokens": 8470, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20496591300002365, "nonpadding_tokens": 11274, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2056272350000654, "nonpadding_tokens": 10083, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20853699999997843, "nonpadding_tokens": 3641, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20515809699998044, "nonpadding_tokens": 5129, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.016639791000102377, "prefetch": {"data_wait_seconds": 0.009847143000982328, "consumed_batches": 1042, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1592, "tokens": 123791432, "loss_tokens": 78615, "loss": 2.9853401466005214, "lr": 0.0002997107238162168, "optimizer_lrs": [0.01998071492108112, 0.0002997107238162168], "gradient_norm": 0.37546563148498535, "seconds": 0.35621873100001267, "nonpadding_tokens_per_second": 220693.05502072885, "ranks": [{"compute_seconds": 0.35036970699991343, "nonpadding_tokens": 4211, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34974968100004844, "nonpadding_tokens": 7358, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34935924100000193, "nonpadding_tokens": 4265, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34960261399999126, "nonpadding_tokens": 13472, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3536111560000563, "nonpadding_tokens": 11258, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35056185100006587, "nonpadding_tokens": 12900, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35314040499997645, "nonpadding_tokens": 17426, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3537751850000177, "nonpadding_tokens": 7725, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003856179999957021, "prefetch": {"data_wait_seconds": 0.009855205000988576, "consumed_batches": 1043, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1593, "tokens": 123836607, "loss_tokens": 45175, "loss": 2.5939601203652463, "lr": 0.0002997104720377957, "optimizer_lrs": [0.01998069813585305, 0.0002997104720377957], "gradient_norm": 0.3817063868045807, "seconds": 0.28096781499993995, "nonpadding_tokens_per_second": 160783.54027848228, "ranks": [{"compute_seconds": 0.2784698500000786, "nonpadding_tokens": 8503, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27578591799999685, "nonpadding_tokens": 4848, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2786012669999991, "nonpadding_tokens": 10426, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2772686309999699, "nonpadding_tokens": 3309, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27506567200009613, "nonpadding_tokens": 5587, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2784188099999483, "nonpadding_tokens": 2336, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2753464959999974, "nonpadding_tokens": 3908, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27580806200001007, "nonpadding_tokens": 6258, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003912380000201665, "prefetch": {"data_wait_seconds": 0.009864947000963298, "consumed_batches": 1044, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1594, "tokens": 123912460, "loss_tokens": 75853, "loss": 2.7048818932672405, "lr": 0.000299710049032481, "optimizer_lrs": [0.019980669935498734, 0.000299710049032481], "gradient_norm": 0.3637795150279999, "seconds": 0.23438433699993766, "nonpadding_tokens_per_second": 323626.574074445, "ranks": [{"compute_seconds": 0.22859954299997298, "nonpadding_tokens": 11218, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23030470399999103, "nonpadding_tokens": 9788, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23002003900001, "nonpadding_tokens": 9297, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22854654999991908, "nonpadding_tokens": 22700, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22674459500001376, "nonpadding_tokens": 3253, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22924412400004712, "nonpadding_tokens": 8404, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23099420699998063, "nonpadding_tokens": 6913, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23205499400000917, "nonpadding_tokens": 4280, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003672469999855821, "prefetch": {"data_wait_seconds": 0.009873833001009302, "consumed_batches": 1045, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1595, "tokens": 123958929, "loss_tokens": 46469, "loss": 2.4435371228668576, "lr": 0.00029970978973896384, "optimizer_lrs": [0.01998065264926426, 0.00029970978973896384], "gradient_norm": 0.5582493543624878, "seconds": 0.23880951800003913, "nonpadding_tokens_per_second": 194586.04660804343, "ranks": [{"compute_seconds": 0.23633456899995053, "nonpadding_tokens": 2871, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2359210060000123, "nonpadding_tokens": 2176, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23492911599998934, "nonpadding_tokens": 2659, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23555758900010915, "nonpadding_tokens": 2416, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2365120099999558, "nonpadding_tokens": 4205, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23316750500009675, "nonpadding_tokens": 13521, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2357828540000355, "nonpadding_tokens": 15857, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23260869199998524, "nonpadding_tokens": 2764, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037936499995794293, "prefetch": {"data_wait_seconds": 0.009882621000954828, "consumed_batches": 1046, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1596, "tokens": 124072668, "loss_tokens": 113739, "loss": 2.579520052488592, "lr": 0.0002997091545954864, "optimizer_lrs": [0.019980610306365762, 0.0002997091545954864], "gradient_norm": 0.3716161251068115, "seconds": 0.33998880499996176, "nonpadding_tokens_per_second": 334537.4857269574, "ranks": [{"compute_seconds": 0.33620627300001615, "nonpadding_tokens": 8186, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3332301540000344, "nonpadding_tokens": 13725, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3359142869999232, "nonpadding_tokens": 26178, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33641515900001195, "nonpadding_tokens": 22822, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33160088600004656, "nonpadding_tokens": 15898, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33465047400000003, "nonpadding_tokens": 11108, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33679741899993587, "nonpadding_tokens": 6992, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33767757900000106, "nonpadding_tokens": 8830, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003344439999182214, "prefetch": {"data_wait_seconds": 0.009891608000955898, "consumed_batches": 1047, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1597, "tokens": 124140377, "loss_tokens": 67709, "loss": 2.6228566364885024, "lr": 0.000299708776164295, "optimizer_lrs": [0.019980585077619666, 0.000299708776164295], "gradient_norm": 0.41223251819610596, "seconds": 0.3178799949999984, "nonpadding_tokens_per_second": 213001.76502142055, "ranks": [{"compute_seconds": 0.31426840000005996, "nonpadding_tokens": 10212, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3130091210000501, "nonpadding_tokens": 4421, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31249951799998144, "nonpadding_tokens": 7390, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3135243570000057, "nonpadding_tokens": 7487, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31146197599991865, "nonpadding_tokens": 9182, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3131461949999448, "nonpadding_tokens": 3586, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31547580799997377, "nonpadding_tokens": 11686, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31258099900003344, "nonpadding_tokens": 13745, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042473100006645836, "prefetch": {"data_wait_seconds": 0.009901631001028477, "consumed_batches": 1048, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1598, "tokens": 124172662, "loss_tokens": 32285, "loss": 2.6226682282793865, "lr": 0.0002997085956342313, "optimizer_lrs": [0.01998057304228209, 0.0002997085956342313], "gradient_norm": 0.4646724760532379, "seconds": 0.2458592599999747, "nonpadding_tokens_per_second": 131314.96450450277, "ranks": [{"compute_seconds": 0.24308421900002486, "nonpadding_tokens": 3830, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2391553700000486, "nonpadding_tokens": 4572, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23972576300002402, "nonpadding_tokens": 2476, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24053381800001716, "nonpadding_tokens": 7845, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23876708399996005, "nonpadding_tokens": 3403, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24166698599992742, "nonpadding_tokens": 4188, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2388402680001036, "nonpadding_tokens": 2770, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24348484300003292, "nonpadding_tokens": 3201, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004344830000491129, "prefetch": {"data_wait_seconds": 0.00991067700108772, "consumed_batches": 1049, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1599, "tokens": 124233389, "loss_tokens": 60727, "loss": 2.2922037355706686, "lr": 0.00029970825591184823, "optimizer_lrs": [0.019980550394123216, 0.00029970825591184823], "gradient_norm": 0.40194815397262573, "seconds": 0.24454386599995814, "nonpadding_tokens_per_second": 248327.6354190393, "ranks": [{"compute_seconds": 0.2400392519999741, "nonpadding_tokens": 5024, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24101921200008292, "nonpadding_tokens": 10225, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24075049000009585, "nonpadding_tokens": 4664, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2415709130000323, "nonpadding_tokens": 11050, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.242195971000001, "nonpadding_tokens": 3158, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.238986961999899, "nonpadding_tokens": 12354, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23710691700000552, "nonpadding_tokens": 9318, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23832314599997062, "nonpadding_tokens": 4934, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003836809999029356, "prefetch": {"data_wait_seconds": 0.009920156001115288, "consumed_batches": 1050, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1600, "tokens": 124313078, "loss_tokens": 79689, "loss": 2.7498752964650075, "lr": 0.000299707809811253, "optimizer_lrs": [0.019980520654083535, 0.000299707809811253], "gradient_norm": 0.39403635263442993, "seconds": 0.31896684000003006, "nonpadding_tokens_per_second": 249834.74771230918, "ranks": [{"compute_seconds": 0.3162537739999607, "nonpadding_tokens": 16713, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31667008499994154, "nonpadding_tokens": 3243, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3154156210000565, "nonpadding_tokens": 13132, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3156441169999198, "nonpadding_tokens": 2493, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3166111049999927, "nonpadding_tokens": 8899, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3152900000000045, "nonpadding_tokens": 4283, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31177575700007765, "nonpadding_tokens": 25598, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31283523599995533, "nonpadding_tokens": 5328, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003756199999997989, "prefetch": {"data_wait_seconds": 0.009928410001137422, "consumed_batches": 1051, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1601, "tokens": 124379289, "loss_tokens": 66211, "loss": 2.144601869402365, "lr": 0.0002997074389018337, "optimizer_lrs": [0.019980495926788914, 0.0002997074389018337], "gradient_norm": 0.5295604467391968, "seconds": 0.3492725449999625, "nonpadding_tokens_per_second": 189568.29257795544, "ranks": [{"compute_seconds": 0.34548707500005094, "nonpadding_tokens": 2530, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3430801900000233, "nonpadding_tokens": 9073, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3408553040000015, "nonpadding_tokens": 10522, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3436295640000253, "nonpadding_tokens": 6361, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3468660239999508, "nonpadding_tokens": 7898, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34232622999991236, "nonpadding_tokens": 12559, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3468313359999229, "nonpadding_tokens": 7030, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3402473670000745, "nonpadding_tokens": 10238, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003603960000191364, "prefetch": {"data_wait_seconds": 0.009937188001188701, "consumed_batches": 1052, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1602, "tokens": 124445362, "loss_tokens": 66073, "loss": 2.73647471357438, "lr": 0.0002997070685312015, "optimizer_lrs": [0.019980471235413435, 0.0002997070685312015], "gradient_norm": 1.3290181159973145, "seconds": 0.41676782599995477, "nonpadding_tokens_per_second": 158536.7100770566, "ranks": [{"compute_seconds": 0.41243127699999604, "nonpadding_tokens": 15833, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4097840450000376, "nonpadding_tokens": 10586, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4124147060000496, "nonpadding_tokens": 4536, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4123371609999822, "nonpadding_tokens": 5434, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.40541321400007746, "nonpadding_tokens": 7593, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4085708860000068, "nonpadding_tokens": 13568, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4060113900000033, "nonpadding_tokens": 2732, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.41424132700001337, "nonpadding_tokens": 5791, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003851289999374785, "prefetch": {"data_wait_seconds": 0.009946042001118371, "consumed_batches": 1053, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1603, "tokens": 124512789, "loss_tokens": 67427, "loss": 2.347734401649191, "lr": 0.0002997066903294771, "optimizer_lrs": [0.01998044602196514, 0.0002997066903294771], "gradient_norm": 0.6381893157958984, "seconds": 0.3069372109999904, "nonpadding_tokens_per_second": 219676.8511068608, "ranks": [{"compute_seconds": 0.3017365839999684, "nonpadding_tokens": 10554, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30341374099998575, "nonpadding_tokens": 4851, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3000605249999353, "nonpadding_tokens": 4321, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3007873840000457, "nonpadding_tokens": 16617, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30458620499996414, "nonpadding_tokens": 11269, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30220721899991076, "nonpadding_tokens": 9957, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2987416649999659, "nonpadding_tokens": 3711, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29980657999999494, "nonpadding_tokens": 6147, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004036549999000272, "prefetch": {"data_wait_seconds": 0.009953776001182746, "consumed_batches": 1054, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1604, "tokens": 124571447, "loss_tokens": 58658, "loss": 2.5393501845443076, "lr": 0.00029970636111531257, "optimizer_lrs": [0.019980424074354172, 0.00029970636111531257], "gradient_norm": 0.3395061492919922, "seconds": 0.24438848599993435, "nonpadding_tokens_per_second": 240019.49093467422, "ranks": [{"compute_seconds": 0.24181530999999268, "nonpadding_tokens": 4732, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2412705570000071, "nonpadding_tokens": 5487, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23814773900005548, "nonpadding_tokens": 16259, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23881846899996617, "nonpadding_tokens": 5432, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24107520600000498, "nonpadding_tokens": 6982, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2394543780000049, "nonpadding_tokens": 2560, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23702514599995084, "nonpadding_tokens": 6993, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24132626799996615, "nonpadding_tokens": 10213, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036222900007487624, "prefetch": {"data_wait_seconds": 0.009960946001186244, "consumed_batches": 1055, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1605, "tokens": 124636606, "loss_tokens": 65159, "loss": 2.214745372857165, "lr": 0.0002997059951984646, "optimizer_lrs": [0.01998039967989764, 0.0002997059951984646], "gradient_norm": 0.4769507050514221, "seconds": 0.25980419500001517, "nonpadding_tokens_per_second": 250800.4152896615, "ranks": [{"compute_seconds": 0.2565578239999695, "nonpadding_tokens": 11357, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2535694019999255, "nonpadding_tokens": 11175, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25416350600005444, "nonpadding_tokens": 17192, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25457250900001327, "nonpadding_tokens": 10349, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2534227410000085, "nonpadding_tokens": 1173, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2544685279999612, "nonpadding_tokens": 2187, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25300984900002277, "nonpadding_tokens": 6940, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2574872359999745, "nonpadding_tokens": 4786, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037250200000471523, "prefetch": {"data_wait_seconds": 0.00996960700122429, "consumed_batches": 1056, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1606, "tokens": 124720426, "loss_tokens": 83820, "loss": 2.631241425077547, "lr": 0.0002997055241513389, "optimizer_lrs": [0.01998036827675593, 0.0002997055241513389], "gradient_norm": 0.5956298112869263, "seconds": 0.2534464029999981, "nonpadding_tokens_per_second": 330720.81121625006, "ranks": [{"compute_seconds": 0.24885248700002194, "nonpadding_tokens": 6253, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24686956700008977, "nonpadding_tokens": 13306, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2494719169999371, "nonpadding_tokens": 3379, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24735904299996037, "nonpadding_tokens": 16949, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25079386600009457, "nonpadding_tokens": 14507, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.248145151000017, "nonpadding_tokens": 15241, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24619274599990604, "nonpadding_tokens": 10966, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25103743299996495, "nonpadding_tokens": 3219, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003522719999864421, "prefetch": {"data_wait_seconds": 0.009978871001138145, "consumed_batches": 1057, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1607, "tokens": 124818976, "loss_tokens": 98550, "loss": 2.280698091070523, "lr": 0.00029970496984361886, "optimizer_lrs": [0.019980331322907927, 0.00029970496984361886], "gradient_norm": 0.6244490146636963, "seconds": 0.3607919819999097, "nonpadding_tokens_per_second": 273149.08566905087, "ranks": [{"compute_seconds": 0.35858905399993546, "nonpadding_tokens": 11270, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3578464839999924, "nonpadding_tokens": 10958, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.357052633999956, "nonpadding_tokens": 21204, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3581578590000163, "nonpadding_tokens": 28967, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35805977699999403, "nonpadding_tokens": 3769, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3569466829999328, "nonpadding_tokens": 11156, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35772627899996223, "nonpadding_tokens": 8560, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3548010999999178, "nonpadding_tokens": 2666, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003417899999931251, "prefetch": {"data_wait_seconds": 0.00998701100111532, "consumed_batches": 1058, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1608, "tokens": 124863356, "loss_tokens": 44380, "loss": 2.3735978692541684, "lr": 0.00029970472005232194, "optimizer_lrs": [0.0199803146701548, 0.00029970472005232194], "gradient_norm": 0.43561190366744995, "seconds": 0.22248260900005334, "nonpadding_tokens_per_second": 199476.26558078235, "ranks": [{"compute_seconds": 0.2190850340000452, "nonpadding_tokens": 2463, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2155428920000304, "nonpadding_tokens": 9192, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2147278230000893, "nonpadding_tokens": 2847, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21519119200002024, "nonpadding_tokens": 4232, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2136167959999966, "nonpadding_tokens": 9269, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21631202800006122, "nonpadding_tokens": 8912, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21302366299994446, "nonpadding_tokens": 4155, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2201323689999981, "nonpadding_tokens": 3310, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033955900005366857, "prefetch": {"data_wait_seconds": 0.009995290001100443, "consumed_batches": 1059, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1609, "tokens": 124914762, "loss_tokens": 51406, "loss": 2.1620912138174533, "lr": 0.0002997044305834463, "optimizer_lrs": [0.019980295372229756, 0.0002997044305834463], "gradient_norm": 0.4203330874443054, "seconds": 0.28674311800000396, "nonpadding_tokens_per_second": 179275.44472052262, "ranks": [{"compute_seconds": 0.2843264039998985, "nonpadding_tokens": 2803, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2840638919999492, "nonpadding_tokens": 5033, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2828435739999122, "nonpadding_tokens": 6097, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2814010929999995, "nonpadding_tokens": 9333, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2840965259999848, "nonpadding_tokens": 5043, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2814261239999496, "nonpadding_tokens": 3447, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2798185060000833, "nonpadding_tokens": 5508, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2804746919999843, "nonpadding_tokens": 14142, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041512500001772423, "prefetch": {"data_wait_seconds": 0.010005048001133332, "consumed_batches": 1060, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1610, "tokens": 124962633, "loss_tokens": 47871, "loss": 2.2977309788285183, "lr": 0.00029970416089288807, "optimizer_lrs": [0.019980277392859205, 0.00029970416089288807], "gradient_norm": 0.43579766154289246, "seconds": 0.22251104100007524, "nonpadding_tokens_per_second": 215139.88602472906, "ranks": [{"compute_seconds": 0.22006888400005664, "nonpadding_tokens": 3248, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21581028699995386, "nonpadding_tokens": 3739, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21894716999997854, "nonpadding_tokens": 5481, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21739385499995478, "nonpadding_tokens": 7854, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21507104199997684, "nonpadding_tokens": 10119, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21785040299994307, "nonpadding_tokens": 7094, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21494239200001175, "nonpadding_tokens": 2072, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2153015650000043, "nonpadding_tokens": 8264, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003504070000417414, "prefetch": {"data_wait_seconds": 0.010014811001113344, "consumed_batches": 1061, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1611, "tokens": 125048758, "loss_tokens": 86125, "loss": 2.3717256894049346, "lr": 0.0002997036753816819, "optimizer_lrs": [0.019980245025445462, 0.0002997036753816819], "gradient_norm": 0.3495118021965027, "seconds": 0.3439780339999743, "nonpadding_tokens_per_second": 250379.35997973185, "ranks": [{"compute_seconds": 0.3412099729999909, "nonpadding_tokens": 2595, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34151324900005875, "nonpadding_tokens": 5084, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3387758339999891, "nonpadding_tokens": 9754, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3416270339999983, "nonpadding_tokens": 16193, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34160798999994313, "nonpadding_tokens": 26259, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3404637069999126, "nonpadding_tokens": 17655, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33735973499995, "nonpadding_tokens": 4391, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33812523299991426, "nonpadding_tokens": 4194, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034831699997539545, "prefetch": {"data_wait_seconds": 0.010023901001162017, "consumed_batches": 1062, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1612, "tokens": 125115395, "loss_tokens": 66637, "loss": 2.928030082761829, "lr": 0.00029970329945704544, "optimizer_lrs": [0.01998021996380303, 0.00029970329945704544], "gradient_norm": 0.4322051703929901, "seconds": 0.3814774689999467, "nonpadding_tokens_per_second": 174681.35188872533, "ranks": [{"compute_seconds": 0.377565789000073, "nonpadding_tokens": 3776, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3754328469999564, "nonpadding_tokens": 10601, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3761627970000063, "nonpadding_tokens": 12624, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3765301509999972, "nonpadding_tokens": 2084, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3791626439999618, "nonpadding_tokens": 15374, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3761262909999914, "nonpadding_tokens": 5717, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37508896500003175, "nonpadding_tokens": 8044, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3757104360000767, "nonpadding_tokens": 8417, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038601999995080405, "prefetch": {"data_wait_seconds": 0.010032182001125989, "consumed_batches": 1063, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1613, "tokens": 125197705, "loss_tokens": 82310, "loss": 2.531842273113838, "lr": 0.0002997028347864563, "optimizer_lrs": [0.019980188985763755, 0.0002997028347864563], "gradient_norm": 0.36501187086105347, "seconds": 0.2809382249999999, "nonpadding_tokens_per_second": 292982.55871019344, "ranks": [{"compute_seconds": 0.27791653600002064, "nonpadding_tokens": 8708, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2785745030000726, "nonpadding_tokens": 7537, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2753143090000094, "nonpadding_tokens": 15946, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27547733100004734, "nonpadding_tokens": 13175, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2743536820000827, "nonpadding_tokens": 3160, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2770984030000818, "nonpadding_tokens": 1734, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2783775330000253, "nonpadding_tokens": 20478, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2748829230000638, "nonpadding_tokens": 11572, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003876880000461824, "prefetch": {"data_wait_seconds": 0.010041526001216994, "consumed_batches": 1064, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1614, "tokens": 125257246, "loss_tokens": 59541, "loss": 2.4408406602173294, "lr": 0.0002997024984289685, "optimizer_lrs": [0.019980166561931235, 0.0002997024984289685], "gradient_norm": 0.7134024500846863, "seconds": 0.2833998830000155, "nonpadding_tokens_per_second": 210095.35843738067, "ranks": [{"compute_seconds": 0.27492584800006625, "nonpadding_tokens": 11723, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2772653030000356, "nonpadding_tokens": 7659, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27938024000002315, "nonpadding_tokens": 5747, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27421813700004805, "nonpadding_tokens": 4231, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28046058799998264, "nonpadding_tokens": 9272, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27773629199998595, "nonpadding_tokens": 5021, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.279768990999969, "nonpadding_tokens": 6749, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28097166999998535, "nonpadding_tokens": 9139, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000372642000002088, "prefetch": {"data_wait_seconds": 0.010050127001207443, "consumed_batches": 1065, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1615, "tokens": 125339633, "loss_tokens": 82387, "loss": 2.386014328716909, "lr": 0.00029970203269703404, "optimizer_lrs": [0.019980135513135603, 0.00029970203269703404], "gradient_norm": 0.3936711251735687, "seconds": 0.29445583999995506, "nonpadding_tokens_per_second": 279794.0771017229, "ranks": [{"compute_seconds": 0.29230890000008003, "nonpadding_tokens": 15869, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28827497100007804, "nonpadding_tokens": 5139, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28852898099989943, "nonpadding_tokens": 13418, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2913555869999982, "nonpadding_tokens": 3916, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2877814920000219, "nonpadding_tokens": 18996, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28945749800004705, "nonpadding_tokens": 1741, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28788560600003166, "nonpadding_tokens": 5367, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2883907430000363, "nonpadding_tokens": 17941, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003383740000799662, "prefetch": {"data_wait_seconds": 0.010058735001123864, "consumed_batches": 1066, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1616, "tokens": 125449338, "loss_tokens": 109705, "loss": 2.6067259924342556, "lr": 0.00029970141197220075, "optimizer_lrs": [0.019980094131480053, 0.00029970141197220075], "gradient_norm": 0.3734714090824127, "seconds": 0.3107639479999307, "nonpadding_tokens_per_second": 353017.13955579064, "ranks": [{"compute_seconds": 0.3071244810000735, "nonpadding_tokens": 7463, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3084258430000091, "nonpadding_tokens": 22626, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3075269710000157, "nonpadding_tokens": 22232, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30756601899997804, "nonpadding_tokens": 14455, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30403001199999835, "nonpadding_tokens": 28003, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3053789499999766, "nonpadding_tokens": 3649, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3080159990000766, "nonpadding_tokens": 3815, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30501145200003066, "nonpadding_tokens": 7462, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003810889999158462, "prefetch": {"data_wait_seconds": 0.010068855001080124, "consumed_batches": 1067, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1617, "tokens": 125518166, "loss_tokens": 68828, "loss": 2.7840731606323006, "lr": 0.0002997010222052717, "optimizer_lrs": [0.019980068147018113, 0.0002997010222052717], "gradient_norm": 0.652051568031311, "seconds": 0.25999675799994293, "nonpadding_tokens_per_second": 264726.37785743125, "ranks": [{"compute_seconds": 0.25652932100001635, "nonpadding_tokens": 10677, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2535185389999697, "nonpadding_tokens": 31560, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25423003100002006, "nonpadding_tokens": 3207, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25661145200001556, "nonpadding_tokens": 5003, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2576508739999781, "nonpadding_tokens": 1390, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25423024100007297, "nonpadding_tokens": 3089, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25324438099994495, "nonpadding_tokens": 5339, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2530849610000132, "nonpadding_tokens": 8563, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003661709999960294, "prefetch": {"data_wait_seconds": 0.010076593001102196, "consumed_batches": 1068, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1618, "tokens": 125577023, "loss_tokens": 58857, "loss": 2.3974731658936066, "lr": 0.00029970068870181326, "optimizer_lrs": [0.01998004591345422, 0.00029970068870181326], "gradient_norm": 0.39374592900276184, "seconds": 0.20189119299993763, "nonpadding_tokens_per_second": 291528.31842455943, "ranks": [{"compute_seconds": 0.19969519699998273, "nonpadding_tokens": 1786, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19565105599997423, "nonpadding_tokens": 2763, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1963193820000697, "nonpadding_tokens": 7248, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.19891367800005355, "nonpadding_tokens": 6784, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1991349759999821, "nonpadding_tokens": 12750, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1979524710000078, "nonpadding_tokens": 13674, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19474270499995328, "nonpadding_tokens": 4418, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.19583762000002025, "nonpadding_tokens": 9434, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038389600001664803, "prefetch": {"data_wait_seconds": 0.010085308001023918, "consumed_batches": 1069, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1619, "tokens": 125630637, "loss_tokens": 53614, "loss": 2.6173440006341626, "lr": 0.0002997003847453257, "optimizer_lrs": [0.01998002564968838, 0.0002997003847453257], "gradient_norm": 0.5550733804702759, "seconds": 0.23721561000002112, "nonpadding_tokens_per_second": 226013.7939488688, "ranks": [{"compute_seconds": 0.23453225499997643, "nonpadding_tokens": 3542, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23140894999994543, "nonpadding_tokens": 5629, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2344684509999979, "nonpadding_tokens": 12003, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23484415100006117, "nonpadding_tokens": 7362, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23493129400003454, "nonpadding_tokens": 9653, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2339375020000034, "nonpadding_tokens": 4262, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23109039700000267, "nonpadding_tokens": 8310, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23161691199993584, "nonpadding_tokens": 2853, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038271699997949327, "prefetch": {"data_wait_seconds": 0.010093291001069815, "consumed_batches": 1070, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1620, "tokens": 125707321, "loss_tokens": 76684, "loss": 2.5473049299723542, "lr": 0.00029969994972914505, "optimizer_lrs": [0.019979996648609673, 0.00029969994972914505], "gradient_norm": 0.7347738742828369, "seconds": 0.33259310500000083, "nonpadding_tokens_per_second": 230564.01003863206, "ranks": [{"compute_seconds": 0.33042271900001197, "nonpadding_tokens": 13633, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.328907357999924, "nonpadding_tokens": 18209, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3293705639999871, "nonpadding_tokens": 5851, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3299418849999256, "nonpadding_tokens": 1709, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32840659100008907, "nonpadding_tokens": 3695, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3299267479999344, "nonpadding_tokens": 10429, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32834187499997824, "nonpadding_tokens": 13667, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3296126090000371, "nonpadding_tokens": 9491, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003646889999799896, "prefetch": {"data_wait_seconds": 0.010101825001086695, "consumed_batches": 1071, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1621, "tokens": 125748662, "loss_tokens": 41341, "loss": 2.3269289642848503, "lr": 0.0002996997150774253, "optimizer_lrs": [0.019979981005161688, 0.0002996997150774253], "gradient_norm": 0.6649383902549744, "seconds": 0.2324356330000228, "nonpadding_tokens_per_second": 177859.99274902893, "ranks": [{"compute_seconds": 0.22937695000007352, "nonpadding_tokens": 4066, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22992354100006196, "nonpadding_tokens": 13409, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22665783100001136, "nonpadding_tokens": 2726, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22961432099998547, "nonpadding_tokens": 4727, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23013989300000048, "nonpadding_tokens": 3473, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2298338350000222, "nonpadding_tokens": 3002, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22593915799996012, "nonpadding_tokens": 3702, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22633285400002023, "nonpadding_tokens": 6236, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003693029999567443, "prefetch": {"data_wait_seconds": 0.010110707001061314, "consumed_batches": 1072, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1622, "tokens": 125785941, "loss_tokens": 37279, "loss": 2.5149164492743905, "lr": 0.0002996995034030823, "optimizer_lrs": [0.01997996689353882, 0.0002996995034030823], "gradient_norm": 0.4332742691040039, "seconds": 0.2509707349998962, "nonpadding_tokens_per_second": 148539.23107813913, "ranks": [{"compute_seconds": 0.2469510099999752, "nonpadding_tokens": 3078, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24525338700004795, "nonpadding_tokens": 2952, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24495902000001024, "nonpadding_tokens": 8866, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24590602700004638, "nonpadding_tokens": 3695, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2445838939999021, "nonpadding_tokens": 2087, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24722000399992794, "nonpadding_tokens": 3539, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24854022700003497, "nonpadding_tokens": 5985, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2450052910000977, "nonpadding_tokens": 7077, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033540800006903737, "prefetch": {"data_wait_seconds": 0.010118992001096672, "consumed_batches": 1073, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1623, "tokens": 125831280, "loss_tokens": 45339, "loss": 2.4065536155407043, "lr": 0.0002996992458627577, "optimizer_lrs": [0.019979949724183845, 0.0002996992458627577], "gradient_norm": 0.44800418615341187, "seconds": 0.23162366700000803, "nonpadding_tokens_per_second": 195744.24577259814, "ranks": [{"compute_seconds": 0.22756287999993674, "nonpadding_tokens": 4469, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22477522300005148, "nonpadding_tokens": 14156, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22596895699996367, "nonpadding_tokens": 2711, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22804995599994982, "nonpadding_tokens": 5874, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22893662199999198, "nonpadding_tokens": 5186, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2276467679999996, "nonpadding_tokens": 6562, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2247890759999791, "nonpadding_tokens": 2691, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22935308000000987, "nonpadding_tokens": 3690, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034248299994033005, "prefetch": {"data_wait_seconds": 0.010127320001018347, "consumed_batches": 1074, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1624, "tokens": 125894756, "loss_tokens": 63476, "loss": 2.864666616122629, "lr": 0.0002996988851132217, "optimizer_lrs": [0.01997992567421478, 0.0002996988851132217], "gradient_norm": 0.44983550906181335, "seconds": 0.30649302400001943, "nonpadding_tokens_per_second": 207104.2243362641, "ranks": [{"compute_seconds": 0.30153119499993863, "nonpadding_tokens": 5006, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30375861499999246, "nonpadding_tokens": 3976, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3031151889999819, "nonpadding_tokens": 9327, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30344699499994476, "nonpadding_tokens": 5202, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2997703820000197, "nonpadding_tokens": 7205, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30279834800001026, "nonpadding_tokens": 5235, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2996509590000187, "nonpadding_tokens": 14125, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30420213699994747, "nonpadding_tokens": 13400, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003402599999162703, "prefetch": {"data_wait_seconds": 0.01013601700101674, "consumed_batches": 1075, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1625, "tokens": 125932357, "loss_tokens": 37601, "loss": 2.798410863407888, "lr": 0.0002996986713157343, "optimizer_lrs": [0.019979911421048952, 0.0002996986713157343], "gradient_norm": 0.4503861665725708, "seconds": 0.2864287710000326, "nonpadding_tokens_per_second": 131275.22025360964, "ranks": [{"compute_seconds": 0.28397674199993617, "nonpadding_tokens": 3252, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28280303999997614, "nonpadding_tokens": 5343, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2796540729999606, "nonpadding_tokens": 9797, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2804528179999579, "nonpadding_tokens": 4015, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28366571300000487, "nonpadding_tokens": 4949, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2820509609999817, "nonpadding_tokens": 3291, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28336104400000295, "nonpadding_tokens": 3491, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28374657499989553, "nonpadding_tokens": 3463, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000367353000001458, "prefetch": {"data_wait_seconds": 0.01014365200099121, "consumed_batches": 1076, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1626, "tokens": 126010314, "loss_tokens": 77957, "loss": 2.3171844734917966, "lr": 0.0002996982278145101, "optimizer_lrs": [0.019979881854300673, 0.0002996982278145101], "gradient_norm": 0.5684388875961304, "seconds": 0.339920156000062, "nonpadding_tokens_per_second": 229339.1510445935, "ranks": [{"compute_seconds": 0.33713788099998965, "nonpadding_tokens": 11448, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33361587599995346, "nonpadding_tokens": 12617, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3336845670000912, "nonpadding_tokens": 2781, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33466059300008055, "nonpadding_tokens": 6495, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3372694720000027, "nonpadding_tokens": 15546, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33557141000005686, "nonpadding_tokens": 14929, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3335368919999837, "nonpadding_tokens": 11019, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33749072099999466, "nonpadding_tokens": 3122, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042856899995058484, "prefetch": {"data_wait_seconds": 0.010154218001005688, "consumed_batches": 1077, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1627, "tokens": 126068727, "loss_tokens": 58413, "loss": 2.4748878674267716, "lr": 0.0002996978952865656, "optimizer_lrs": [0.019979859685771043, 0.0002996978952865656], "gradient_norm": 1.051910638809204, "seconds": 0.3012112819999402, "nonpadding_tokens_per_second": 193926.99905580428, "ranks": [{"compute_seconds": 0.2988585639999428, "nonpadding_tokens": 10638, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.298841019000065, "nonpadding_tokens": 9980, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2956994690000556, "nonpadding_tokens": 12955, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2958950029999414, "nonpadding_tokens": 5611, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2986627819999512, "nonpadding_tokens": 6218, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2969871020000028, "nonpadding_tokens": 3834, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2945612749999782, "nonpadding_tokens": 2251, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29519277100007457, "nonpadding_tokens": 6926, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036034300001119846, "prefetch": {"data_wait_seconds": 0.010163410000998852, "consumed_batches": 1078, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1628, "tokens": 126153730, "loss_tokens": 85003, "loss": 2.3302951439949178, "lr": 0.00029969741106286916, "optimizer_lrs": [0.01997982740419128, 0.00029969741106286916], "gradient_norm": 0.44950973987579346, "seconds": 0.4026486750000231, "nonpadding_tokens_per_second": 211109.5982123749, "ranks": [{"compute_seconds": 0.37431857100000343, "nonpadding_tokens": 6368, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3788579590000154, "nonpadding_tokens": 6189, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37552550399993834, "nonpadding_tokens": 3200, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.37843272299994624, "nonpadding_tokens": 6712, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3790466489999744, "nonpadding_tokens": 10570, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37765046699996674, "nonpadding_tokens": 21166, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37893856799996684, "nonpadding_tokens": 21220, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3752038419999053, "nonpadding_tokens": 9578, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.012480244000016683, "prefetch": {"data_wait_seconds": 0.010172753000986745, "consumed_batches": 1079, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1629, "tokens": 126217224, "loss_tokens": 63494, "loss": 2.8262948566006236, "lr": 0.0002996970491135214, "optimizer_lrs": [0.01997980327423476, 0.0002996970491135214], "gradient_norm": 0.41794154047966003, "seconds": 0.27626911900006235, "nonpadding_tokens_per_second": 229826.62785407322, "ranks": [{"compute_seconds": 0.2740049799999724, "nonpadding_tokens": 4396, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2703851310000118, "nonpadding_tokens": 9433, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2729580459999852, "nonpadding_tokens": 4729, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2711384140000064, "nonpadding_tokens": 6658, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2703326460000426, "nonpadding_tokens": 3202, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2712958829999934, "nonpadding_tokens": 11145, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2697910539999384, "nonpadding_tokens": 9597, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2700198189999128, "nonpadding_tokens": 14334, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034448700000666577, "prefetch": {"data_wait_seconds": 0.010181279000974541, "consumed_batches": 1080, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1630, "tokens": 126271588, "loss_tokens": 54364, "loss": 2.4786462548745494, "lr": 0.00029969673903827005, "optimizer_lrs": [0.019979782602551337, 0.00029969673903827005], "gradient_norm": 0.4087732136249542, "seconds": 0.3217223130000093, "nonpadding_tokens_per_second": 168978.0217388852, "ranks": [{"compute_seconds": 0.31926431599993066, "nonpadding_tokens": 16804, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31941693200008103, "nonpadding_tokens": 3911, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31833976699999766, "nonpadding_tokens": 12711, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3188204319999386, "nonpadding_tokens": 4614, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31503959199994824, "nonpadding_tokens": 6376, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31811940600005073, "nonpadding_tokens": 5415, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.318903889000012, "nonpadding_tokens": 1682, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31586009499994816, "nonpadding_tokens": 2851, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003902669999433783, "prefetch": {"data_wait_seconds": 0.01019018700101526, "consumed_batches": 1081, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1631, "tokens": 126344576, "loss_tokens": 72988, "loss": 2.781616070449937, "lr": 0.0002996963224884187, "optimizer_lrs": [0.019979754832561248, 0.0002996963224884187], "gradient_norm": 0.41479620337486267, "seconds": 0.30963826799995786, "nonpadding_tokens_per_second": 235720.21788989575, "ranks": [{"compute_seconds": 0.3064278419999482, "nonpadding_tokens": 7578, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3054467360000217, "nonpadding_tokens": 11699, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30308669099997587, "nonpadding_tokens": 4071, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30386500700001307, "nonpadding_tokens": 20076, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3072274109999853, "nonpadding_tokens": 6483, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30436540899995634, "nonpadding_tokens": 5586, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3070641890000161, "nonpadding_tokens": 5276, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30254038699990815, "nonpadding_tokens": 12219, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039680999998381594, "prefetch": {"data_wait_seconds": 0.010198225001047945, "consumed_batches": 1082, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1632, "tokens": 126440795, "loss_tokens": 96219, "loss": 2.424741994824307, "lr": 0.00029969577292055465, "optimizer_lrs": [0.019979718194703646, 0.00029969577292055465], "gradient_norm": 0.46803638339042664, "seconds": 0.27855116500006716, "nonpadding_tokens_per_second": 345426.6651513621, "ranks": [{"compute_seconds": 0.2751917569999023, "nonpadding_tokens": 6450, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27279887500003497, "nonpadding_tokens": 2106, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2734335180000471, "nonpadding_tokens": 2643, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27428233799992086, "nonpadding_tokens": 13007, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2762796590000107, "nonpadding_tokens": 3053, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2734238670000195, "nonpadding_tokens": 24424, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27619449300004817, "nonpadding_tokens": 15333, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27281784800004516, "nonpadding_tokens": 29203, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003813509999872622, "prefetch": {"data_wait_seconds": 0.01020804500110728, "consumed_batches": 1083, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1633, "tokens": 126510266, "loss_tokens": 69471, "loss": 2.4413530016121836, "lr": 0.00029969537581901214, "optimizer_lrs": [0.019979691721267477, 0.00029969537581901214], "gradient_norm": 0.41417986154556274, "seconds": 0.32466161100001045, "nonpadding_tokens_per_second": 213979.71810100382, "ranks": [{"compute_seconds": 0.32186988700004804, "nonpadding_tokens": 4355, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31877880100000766, "nonpadding_tokens": 7351, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3189448440000433, "nonpadding_tokens": 11233, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3223650649999854, "nonpadding_tokens": 10984, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31862101100000473, "nonpadding_tokens": 3975, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3210302389999242, "nonpadding_tokens": 8855, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32199233400001503, "nonpadding_tokens": 9405, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31901484299999083, "nonpadding_tokens": 13313, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003579980000267824, "prefetch": {"data_wait_seconds": 0.010216643001015768, "consumed_batches": 1084, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1634, "tokens": 126597577, "loss_tokens": 87311, "loss": 2.585255222709624, "lr": 0.00029969487637578285, "optimizer_lrs": [0.019979658425052192, 0.00029969487637578285], "gradient_norm": 0.41010332107543945, "seconds": 0.374983119000035, "nonpadding_tokens_per_second": 232839.8148504169, "ranks": [{"compute_seconds": 0.3700679459999492, "nonpadding_tokens": 10729, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36801712999999836, "nonpadding_tokens": 4338, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3688151820000485, "nonpadding_tokens": 12780, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3713402239999368, "nonpadding_tokens": 3180, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36743697800000064, "nonpadding_tokens": 15461, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3707788660000233, "nonpadding_tokens": 18095, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3676812730000165, "nonpadding_tokens": 20722, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3726624889999357, "nonpadding_tokens": 2006, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037052199991194357, "prefetch": {"data_wait_seconds": 0.010225232001019435, "consumed_batches": 1085, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1635, "tokens": 126689885, "loss_tokens": 92308, "loss": 2.3224879614984615, "lr": 0.0002996943479039432, "optimizer_lrs": [0.019979623193596214, 0.0002996943479039432], "gradient_norm": 0.38095974922180176, "seconds": 0.25792822799996884, "nonpadding_tokens_per_second": 357882.5036552849, "ranks": [{"compute_seconds": 0.2556648369999266, "nonpadding_tokens": 5290, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2548951879999777, "nonpadding_tokens": 6080, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2521023919999834, "nonpadding_tokens": 14447, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25494597099998373, "nonpadding_tokens": 17965, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2556304040000441, "nonpadding_tokens": 4311, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2537863000000016, "nonpadding_tokens": 6245, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25515360500003226, "nonpadding_tokens": 15509, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2520883160000267, "nonpadding_tokens": 22461, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003573379999579629, "prefetch": {"data_wait_seconds": 0.010234493000965017, "consumed_batches": 1086, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1636, "tokens": 126757483, "loss_tokens": 67598, "loss": 2.6629557734696294, "lr": 0.00029969396060943235, "optimizer_lrs": [0.019979597373962158, 0.00029969396060943235], "gradient_norm": 0.42466938495635986, "seconds": 0.2522556500000519, "nonpadding_tokens_per_second": 267974.1761977823, "ranks": [{"compute_seconds": 0.24874352000006184, "nonpadding_tokens": 2209, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2461636619999581, "nonpadding_tokens": 7214, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24639325200007534, "nonpadding_tokens": 13647, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24726586999997835, "nonpadding_tokens": 2614, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2457328809999808, "nonpadding_tokens": 15129, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24822575700000016, "nonpadding_tokens": 18392, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24556957599997986, "nonpadding_tokens": 5426, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2499188750000485, "nonpadding_tokens": 2967, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003970390000631596, "prefetch": {"data_wait_seconds": 0.010244518000902758, "consumed_batches": 1087, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1637, "tokens": 126851765, "loss_tokens": 94282, "loss": 2.608911298020831, "lr": 0.000299693420023138, "optimizer_lrs": [0.019979561334875866, 0.000299693420023138], "gradient_norm": 0.4899997115135193, "seconds": 0.2829147230000899, "nonpadding_tokens_per_second": 333252.3631156871, "ranks": [{"compute_seconds": 0.27743872499991085, "nonpadding_tokens": 8834, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27992145099995014, "nonpadding_tokens": 15709, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2788429449999512, "nonpadding_tokens": 10756, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27735324900004343, "nonpadding_tokens": 4464, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2798748440000054, "nonpadding_tokens": 3421, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27808204600000863, "nonpadding_tokens": 11256, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2754126150000502, "nonpadding_tokens": 16393, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2805853779999552, "nonpadding_tokens": 23449, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003878689999510243, "prefetch": {"data_wait_seconds": 0.010255068000859069, "consumed_batches": 1088, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1638, "tokens": 126913899, "loss_tokens": 62134, "loss": 2.7123349333698137, "lr": 0.00029969306350389747, "optimizer_lrs": [0.0199795375669265, 0.00029969306350389747], "gradient_norm": 0.6273501515388489, "seconds": 0.29327070800002275, "nonpadding_tokens_per_second": 211865.6869065668, "ranks": [{"compute_seconds": 0.28918516399994587, "nonpadding_tokens": 11463, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2868976859999748, "nonpadding_tokens": 2503, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28774551899994094, "nonpadding_tokens": 7530, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2901936650000607, "nonpadding_tokens": 10839, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2866810819999728, "nonpadding_tokens": 4201, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2892263249999587, "nonpadding_tokens": 2849, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2903506809999499, "nonpadding_tokens": 11237, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2909656609999729, "nonpadding_tokens": 11512, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003422579999323716, "prefetch": {"data_wait_seconds": 0.010263483000883866, "consumed_batches": 1089, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1639, "tokens": 126971420, "loss_tokens": 57521, "loss": 2.4644246775090837, "lr": 0.00029969273326916647, "optimizer_lrs": [0.019979515551277766, 0.00029969273326916647], "gradient_norm": 0.37680572271347046, "seconds": 0.8836813689999872, "nonpadding_tokens_per_second": 65092.46660376387, "ranks": [{"compute_seconds": 0.8811423199999808, "nonpadding_tokens": 9561, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.8799828830000251, "nonpadding_tokens": 3500, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.8807544289999214, "nonpadding_tokens": 13339, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.8777712290000181, "nonpadding_tokens": 1886, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.8755023229999779, "nonpadding_tokens": 7292, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.8786112630000389, "nonpadding_tokens": 1977, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.8812033529999326, "nonpadding_tokens": 17502, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.8765015349999885, "nonpadding_tokens": 2464, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039776599999186146, "prefetch": {"data_wait_seconds": 0.010272198000919275, "consumed_batches": 1090, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1640, "tokens": 127022191, "loss_tokens": 50771, "loss": 2.5599114775166925, "lr": 0.0002996924416396124, "optimizer_lrs": [0.019979496109307496, 0.0002996924416396124], "gradient_norm": 0.4222750961780548, "seconds": 0.19389882699999816, "nonpadding_tokens_per_second": 261842.7392549439, "ranks": [{"compute_seconds": 0.19129579699995247, "nonpadding_tokens": 2823, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19032807999997203, "nonpadding_tokens": 7580, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.18758430299999418, "nonpadding_tokens": 3431, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.18790456500005348, "nonpadding_tokens": 11089, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.19156928300003528, "nonpadding_tokens": 3214, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.18951850200005538, "nonpadding_tokens": 5451, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.1861763690000089, "nonpadding_tokens": 2464, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.18726014600008511, "nonpadding_tokens": 14719, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037016399994627136, "prefetch": {"data_wait_seconds": 0.01028186200092307, "consumed_batches": 1091, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1641, "tokens": 127078353, "loss_tokens": 56162, "loss": 2.301059602133115, "lr": 0.00029969211888309906, "optimizer_lrs": [0.019979474592206604, 0.00029969211888309906], "gradient_norm": 0.5844296216964722, "seconds": 0.2269102699999621, "nonpadding_tokens_per_second": 247507.52797574733, "ranks": [{"compute_seconds": 0.2237609239999756, "nonpadding_tokens": 2099, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22373451900000418, "nonpadding_tokens": 3110, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22059101700006067, "nonpadding_tokens": 6049, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2208672059999799, "nonpadding_tokens": 11435, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21931162200007748, "nonpadding_tokens": 7670, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2212067880000177, "nonpadding_tokens": 7270, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22367074800001774, "nonpadding_tokens": 8169, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22461294499998985, "nonpadding_tokens": 10360, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000351184999999532, "prefetch": {"data_wait_seconds": 0.010290192000937282, "consumed_batches": 1092, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1642, "tokens": 127120300, "loss_tokens": 41947, "loss": 2.416409397573128, "lr": 0.00029969187770824846, "optimizer_lrs": [0.019979458513883233, 0.00029969187770824846], "gradient_norm": 0.4814947843551636, "seconds": 0.3523602770000025, "nonpadding_tokens_per_second": 119045.76860120843, "ranks": [{"compute_seconds": 0.34967902100004267, "nonpadding_tokens": 9614, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.349666845999991, "nonpadding_tokens": 2761, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34662550300004114, "nonpadding_tokens": 2014, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34926987700009704, "nonpadding_tokens": 5084, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3499707099999796, "nonpadding_tokens": 4524, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34830515099997683, "nonpadding_tokens": 4227, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3457701080000106, "nonpadding_tokens": 2143, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3500250980000601, "nonpadding_tokens": 11580, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036923999994087353, "prefetch": {"data_wait_seconds": 0.010299535000925175, "consumed_batches": 1093, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1643, "tokens": 127235643, "loss_tokens": 115343, "loss": 2.6711070459412363, "lr": 0.0002996912140558585, "optimizer_lrs": [0.019979414270390565, 0.0002996912140558585], "gradient_norm": 0.6841148138046265, "seconds": 0.2800407920000225, "nonpadding_tokens_per_second": 411879.28078703163, "ranks": [{"compute_seconds": 0.2775446200000715, "nonpadding_tokens": 11717, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27349437999998827, "nonpadding_tokens": 10373, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27020055800005593, "nonpadding_tokens": 5168, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2763257249999924, "nonpadding_tokens": 2209, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26899241099999927, "nonpadding_tokens": 21808, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27208896000001914, "nonpadding_tokens": 21111, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26914071499993497, "nonpadding_tokens": 25363, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2699011749999727, "nonpadding_tokens": 17594, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003995350000423059, "prefetch": {"data_wait_seconds": 0.010307632000902345, "consumed_batches": 1094, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1644, "tokens": 127356705, "loss_tokens": 121062, "loss": 3.0391836930663625, "lr": 0.00029969051673085675, "optimizer_lrs": [0.01997936778205712, 0.00029969051673085675], "gradient_norm": 1.5258365869522095, "seconds": 0.35120856300000014, "nonpadding_tokens_per_second": 344701.1626535995, "ranks": [{"compute_seconds": 0.3421765630000664, "nonpadding_tokens": 20234, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3445181170000069, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3407256479999887, "nonpadding_tokens": 11900, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3411115389999395, "nonpadding_tokens": 11273, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33954753399996207, "nonpadding_tokens": 10523, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3451835719999963, "nonpadding_tokens": 11991, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3479568329998983, "nonpadding_tokens": 12381, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3486378220000006, "nonpadding_tokens": 9992, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004458839999870179, "prefetch": {"data_wait_seconds": 0.010319058000803238, "consumed_batches": 1095, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1645, "tokens": 127456039, "loss_tokens": 99334, "loss": 2.8157208256991564, "lr": 0.000299689943973804, "optimizer_lrs": [0.0199793295982536, 0.000299689943973804], "gradient_norm": 1.5471118688583374, "seconds": 0.24449247699999432, "nonpadding_tokens_per_second": 406286.5296260314, "ranks": [{"compute_seconds": 0.23990493600001628, "nonpadding_tokens": 4425, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23868453000000045, "nonpadding_tokens": 14282, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23598731199990652, "nonpadding_tokens": 10036, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24190635400009342, "nonpadding_tokens": 13214, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23466783699996085, "nonpadding_tokens": 7157, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23980664599991997, "nonpadding_tokens": 12469, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24202415299998847, "nonpadding_tokens": 19565, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2355713529999548, "nonpadding_tokens": 18186, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041408599997794227, "prefetch": {"data_wait_seconds": 0.010329306000770089, "consumed_batches": 1096, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1646, "tokens": 127598943, "loss_tokens": 142904, "loss": 2.8792248642445277, "lr": 0.00029968911906560206, "optimizer_lrs": [0.019979274604373473, 0.00029968911906560206], "gradient_norm": 1.0921995639801025, "seconds": 0.3292926469999884, "nonpadding_tokens_per_second": 433972.6419703658, "ranks": [{"compute_seconds": 0.32487817599997015, "nonpadding_tokens": 13958, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3236081249999643, "nonpadding_tokens": 13894, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32626615399999537, "nonpadding_tokens": 17290, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3218092859999615, "nonpadding_tokens": 20236, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3200556110000434, "nonpadding_tokens": 25106, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32469745599996713, "nonpadding_tokens": 17591, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32677676399998745, "nonpadding_tokens": 9405, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3209202360000063, "nonpadding_tokens": 25424, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003914490000624937, "prefetch": {"data_wait_seconds": 0.010339013000773889, "consumed_batches": 1097, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1647, "tokens": 127718559, "loss_tokens": 119616, "loss": 2.948738567582932, "lr": 0.00029968842774496276, "optimizer_lrs": [0.01997922851633085, 0.00029968842774496276], "gradient_norm": 1.3137052059173584, "seconds": 0.2924976400000787, "nonpadding_tokens_per_second": 408946.8892807744, "ranks": [{"compute_seconds": 0.2600898230000439, "nonpadding_tokens": 12352, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2649208829999452, "nonpadding_tokens": 16773, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2657035770000675, "nonpadding_tokens": 25058, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2682469100000162, "nonpadding_tokens": 9667, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2645078680000097, "nonpadding_tokens": 15774, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2677840459999743, "nonpadding_tokens": 8520, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26463581500001965, "nonpadding_tokens": 12047, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26962008199996035, "nonpadding_tokens": 19425, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.014901958999985254, "prefetch": {"data_wait_seconds": 0.010346826000798615, "consumed_batches": 1098, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1648, "tokens": 127778822, "loss_tokens": 60263, "loss": 2.5980185291970197, "lr": 0.0002996880791644528, "optimizer_lrs": [0.01997920527763019, 0.0002996880791644528], "gradient_norm": 0.47671690583229065, "seconds": 0.3101276109999844, "nonpadding_tokens_per_second": 194316.78400283755, "ranks": [{"compute_seconds": 0.3057657190000782, "nonpadding_tokens": 1610, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3076506109999855, "nonpadding_tokens": 12552, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3068388789999972, "nonpadding_tokens": 10849, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3069281550000369, "nonpadding_tokens": 14461, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30783640800007106, "nonpadding_tokens": 1635, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3051782700000558, "nonpadding_tokens": 6958, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30750447499997335, "nonpadding_tokens": 3930, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3043549079999366, "nonpadding_tokens": 8268, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003424369999720511, "prefetch": {"data_wait_seconds": 0.010354502000794241, "consumed_batches": 1099, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1649, "tokens": 127827425, "loss_tokens": 48603, "loss": 2.520873197127749, "lr": 0.0002996877978873143, "optimizer_lrs": [0.019979186525820956, 0.0002996877978873143], "gradient_norm": 0.4199119210243225, "seconds": 0.28703754399998616, "nonpadding_tokens_per_second": 169326.28158218335, "ranks": [{"compute_seconds": 0.28306471500002317, "nonpadding_tokens": 3392, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28464695899992876, "nonpadding_tokens": 3933, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.284685272000047, "nonpadding_tokens": 3405, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28429274200004784, "nonpadding_tokens": 7370, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28113601699999435, "nonpadding_tokens": 11151, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28221451699994304, "nonpadding_tokens": 10736, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28477384299992536, "nonpadding_tokens": 4811, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.281996821000007, "nonpadding_tokens": 3805, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035229399998115696, "prefetch": {"data_wait_seconds": 0.010362212000771365, "consumed_batches": 1100, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1650, "tokens": 127895435, "loss_tokens": 68010, "loss": 2.634348579253051, "lr": 0.00029968740408469494, "optimizer_lrs": [0.019979160272312998, 0.00029968740408469494], "gradient_norm": 0.37528151273727417, "seconds": 0.23319898800002647, "nonpadding_tokens_per_second": 291639.3445068993, "ranks": [{"compute_seconds": 0.23049979599989, "nonpadding_tokens": 2349, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23082608799995796, "nonpadding_tokens": 14564, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2276907199999414, "nonpadding_tokens": 8330, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23065814699998555, "nonpadding_tokens": 12656, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22680980399991313, "nonpadding_tokens": 15643, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22997399899998072, "nonpadding_tokens": 4197, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23088920299994697, "nonpadding_tokens": 6947, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22779694799999106, "nonpadding_tokens": 3324, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038500000005115, "prefetch": {"data_wait_seconds": 0.01037205300083599, "consumed_batches": 1101, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1651, "tokens": 127952794, "loss_tokens": 57359, "loss": 2.4444896942938334, "lr": 0.00029968707176251556, "optimizer_lrs": [0.019979138117501037, 0.00029968707176251556], "gradient_norm": 0.42151254415512085, "seconds": 0.29479181399995014, "nonpadding_tokens_per_second": 194574.6024006274, "ranks": [{"compute_seconds": 0.2803610749999734, "nonpadding_tokens": 7006, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2771008309999843, "nonpadding_tokens": 3259, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2802436910000097, "nonpadding_tokens": 8736, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27856151299999965, "nonpadding_tokens": 15301, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2807896990000245, "nonpadding_tokens": 6708, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2796126739999636, "nonpadding_tokens": 9274, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27657883500000935, "nonpadding_tokens": 5107, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27744894199997816, "nonpadding_tokens": 1968, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.004218594000008125, "prefetch": {"data_wait_seconds": 0.010380897000757727, "consumed_batches": 1102, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1652, "tokens": 128012734, "loss_tokens": 59940, "loss": 2.85103019686353, "lr": 0.00029968672429830695, "optimizer_lrs": [0.019979114953220464, 0.00029968672429830695], "gradient_norm": 0.41631758213043213, "seconds": 0.3223069859999441, "nonpadding_tokens_per_second": 185971.76792193513, "ranks": [{"compute_seconds": 0.31794191699998464, "nonpadding_tokens": 6258, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31580432100008693, "nonpadding_tokens": 7220, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3173109839999597, "nonpadding_tokens": 1398, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3189026620000277, "nonpadding_tokens": 7146, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3155438000000004, "nonpadding_tokens": 10489, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31761076399993726, "nonpadding_tokens": 15179, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3153910300000007, "nonpadding_tokens": 8538, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3200134029999617, "nonpadding_tokens": 3712, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035176100004719046, "prefetch": {"data_wait_seconds": 0.010388499000740126, "consumed_batches": 1103, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1653, "tokens": 128078863, "loss_tokens": 66129, "loss": 2.3351997138169334, "lr": 0.00029968634073386123, "optimizer_lrs": [0.019979089382257415, 0.00029968634073386123], "gradient_norm": 0.43470120429992676, "seconds": 0.32329304699999284, "nonpadding_tokens_per_second": 204548.16648129604, "ranks": [{"compute_seconds": 0.3179330339999069, "nonpadding_tokens": 9663, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3182958919999237, "nonpadding_tokens": 12347, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31482624899990697, "nonpadding_tokens": 3182, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32014743700005965, "nonpadding_tokens": 9404, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3207925450000175, "nonpadding_tokens": 9675, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3177696699999615, "nonpadding_tokens": 9224, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3205351579999842, "nonpadding_tokens": 8579, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3146189569999933, "nonpadding_tokens": 4055, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038756999992983765, "prefetch": {"data_wait_seconds": 0.010396856000809862, "consumed_batches": 1104, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1654, "tokens": 128116491, "loss_tokens": 37628, "loss": 2.7078698459259063, "lr": 0.00029968612237758557, "optimizer_lrs": [0.01997907482517237, 0.00029968612237758557], "gradient_norm": 0.44522982835769653, "seconds": 0.22444392200009133, "nonpadding_tokens_per_second": 167649.89519290565, "ranks": [{"compute_seconds": 0.22044626100000642, "nonpadding_tokens": 2505, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2212984889999916, "nonpadding_tokens": 3513, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2208762940000497, "nonpadding_tokens": 1247, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21930940400000054, "nonpadding_tokens": 5985, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22196139599998332, "nonpadding_tokens": 10414, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21912454499999967, "nonpadding_tokens": 3922, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21717247700007647, "nonpadding_tokens": 3953, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21786365700006627, "nonpadding_tokens": 6089, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004402309999704812, "prefetch": {"data_wait_seconds": 0.010405263000805576, "consumed_batches": 1105, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1655, "tokens": 128169675, "loss_tokens": 53184, "loss": 2.6484836252444346, "lr": 0.000299685813619995, "optimizer_lrs": [0.019979054241333002, 0.000299685813619995], "gradient_norm": 0.4069131910800934, "seconds": 0.2788932329999625, "nonpadding_tokens_per_second": 190696.63120871474, "ranks": [{"compute_seconds": 0.2762182840000378, "nonpadding_tokens": 3438, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27397840600008294, "nonpadding_tokens": 2842, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.270614476999981, "nonpadding_tokens": 6230, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27128976300002705, "nonpadding_tokens": 2453, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27593471000000136, "nonpadding_tokens": 2771, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27271123000002717, "nonpadding_tokens": 9252, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27549831399994673, "nonpadding_tokens": 21353, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2763640759999362, "nonpadding_tokens": 4845, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036333200000626675, "prefetch": {"data_wait_seconds": 0.01041416700077491, "consumed_batches": 1106, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1656, "tokens": 128237805, "loss_tokens": 68130, "loss": 2.6614214094378394, "lr": 0.0002996854178724934, "optimizer_lrs": [0.01997902785816623, 0.0002996854178724934], "gradient_norm": 0.43997547030448914, "seconds": 0.48629069300000083, "nonpadding_tokens_per_second": 140101.38581862574, "ranks": [{"compute_seconds": 0.48378933699996196, "nonpadding_tokens": 5296, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4800894069999231, "nonpadding_tokens": 7507, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4776480509999601, "nonpadding_tokens": 10328, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4794715790000055, "nonpadding_tokens": 9969, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.48354125399998793, "nonpadding_tokens": 3733, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.48100644299995565, "nonpadding_tokens": 2458, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.47641385199995057, "nonpadding_tokens": 11295, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.47721285199997965, "nonpadding_tokens": 17544, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003952410000920281, "prefetch": {"data_wait_seconds": 0.010422755000789152, "consumed_batches": 1107, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1657, "tokens": 128318197, "loss_tokens": 80392, "loss": 2.8547842136033434, "lr": 0.00029968495057858453, "optimizer_lrs": [0.01997899670523897, 0.00029968495057858453], "gradient_norm": 0.5639077425003052, "seconds": 0.3241014919999543, "nonpadding_tokens_per_second": 248045.75722228189, "ranks": [{"compute_seconds": 0.32177828100009265, "nonpadding_tokens": 18833, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32110740100006296, "nonpadding_tokens": 4586, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.317598650999912, "nonpadding_tokens": 15884, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31825236099996346, "nonpadding_tokens": 11548, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32093802999997934, "nonpadding_tokens": 5059, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3193055940000704, "nonpadding_tokens": 4919, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3161363020000181, "nonpadding_tokens": 10488, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3210403379999889, "nonpadding_tokens": 9075, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040296300005593366, "prefetch": {"data_wait_seconds": 0.010433417000811573, "consumed_batches": 1108, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1658, "tokens": 128371468, "loss_tokens": 53271, "loss": 2.633119450545325, "lr": 0.0002996846407398628, "optimizer_lrs": [0.019978976049324186, 0.0002996846407398628], "gradient_norm": 0.42177119851112366, "seconds": 0.2711735469999894, "nonpadding_tokens_per_second": 196446.1526182791, "ranks": [{"compute_seconds": 0.2650960229999555, "nonpadding_tokens": 5324, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2649542039999915, "nonpadding_tokens": 4642, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26516569000000345, "nonpadding_tokens": 9929, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2654866249999941, "nonpadding_tokens": 7616, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2666109150000011, "nonpadding_tokens": 6085, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2639047060000621, "nonpadding_tokens": 2265, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2656773420000036, "nonpadding_tokens": 7336, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26202680799997324, "nonpadding_tokens": 10074, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004770030000145198, "prefetch": {"data_wait_seconds": 0.010441625000751742, "consumed_batches": 1109, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1659, "tokens": 128449042, "loss_tokens": 77574, "loss": 2.704004402248176, "lr": 0.0002996841892762179, "optimizer_lrs": [0.019978945951747863, 0.0002996841892762179], "gradient_norm": 0.46880021691322327, "seconds": 0.31630100199993194, "nonpadding_tokens_per_second": 245253.72828258283, "ranks": [{"compute_seconds": 0.31361479899999267, "nonpadding_tokens": 6763, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30997541900001124, "nonpadding_tokens": 3296, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3106098880000445, "nonpadding_tokens": 3432, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3132772559999921, "nonpadding_tokens": 9391, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3096895820000327, "nonpadding_tokens": 17300, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3121988189999456, "nonpadding_tokens": 11121, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30940058499993484, "nonpadding_tokens": 18504, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3139954250000301, "nonpadding_tokens": 7767, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034877899997809436, "prefetch": {"data_wait_seconds": 0.010449066000731477, "consumed_batches": 1110, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1660, "tokens": 128567202, "loss_tokens": 118160, "loss": 2.575189626354096, "lr": 0.0002996835009911477, "optimizer_lrs": [0.019978900066076513, 0.0002996835009911477], "gradient_norm": 0.4248030483722687, "seconds": 0.3952966760000436, "nonpadding_tokens_per_second": 298914.7320833706, "ranks": [{"compute_seconds": 0.39274609699998564, "nonpadding_tokens": 14012, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3921755180000446, "nonpadding_tokens": 17430, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3897068449999779, "nonpadding_tokens": 16034, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3905565900000738, "nonpadding_tokens": 24618, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3886419840000599, "nonpadding_tokens": 24050, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39024094200010495, "nonpadding_tokens": 4326, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3925780870000608, "nonpadding_tokens": 11426, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3929789389999314, "nonpadding_tokens": 6264, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003861760000063441, "prefetch": {"data_wait_seconds": 0.01045642700064775, "consumed_batches": 1111, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1661, "tokens": 128631601, "loss_tokens": 64399, "loss": 2.487396446373391, "lr": 0.0002996831255502285, "optimizer_lrs": [0.0199788750366819, 0.0002996831255502285], "gradient_norm": 0.4973495900630951, "seconds": 0.25108474500007105, "nonpadding_tokens_per_second": 256483.1248508617, "ranks": [{"compute_seconds": 0.24819655500004956, "nonpadding_tokens": 4206, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24503325900002437, "nonpadding_tokens": 3325, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2455404910000425, "nonpadding_tokens": 27659, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24835776200006876, "nonpadding_tokens": 2769, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24878649900006167, "nonpadding_tokens": 3472, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24784281200004443, "nonpadding_tokens": 11705, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2449073050000834, "nonpadding_tokens": 5267, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24879611100004695, "nonpadding_tokens": 5996, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038141599998198217, "prefetch": {"data_wait_seconds": 0.010463689000587806, "consumed_batches": 1112, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1662, "tokens": 128706816, "loss_tokens": 75215, "loss": 2.4286482915641825, "lr": 0.0002996826867714895, "optimizer_lrs": [0.01997884578476597, 0.0002996826867714895], "gradient_norm": 0.42850324511528015, "seconds": 0.2850395740000522, "nonpadding_tokens_per_second": 263875.63994881016, "ranks": [{"compute_seconds": 0.28220911400001114, "nonpadding_tokens": 12285, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28155086699996446, "nonpadding_tokens": 4473, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27850027600004523, "nonpadding_tokens": 10433, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2821698530000276, "nonpadding_tokens": 6302, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27741487599996617, "nonpadding_tokens": 5858, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2799269480000248, "nonpadding_tokens": 8914, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28236646200002724, "nonpadding_tokens": 11601, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2827789020000182, "nonpadding_tokens": 15349, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003741750000472166, "prefetch": {"data_wait_seconds": 0.010471033000570173, "consumed_batches": 1113, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1663, "tokens": 128768050, "loss_tokens": 61234, "loss": 2.7397544664728746, "lr": 0.00029968232932923495, "optimizer_lrs": [0.019978821955282333, 0.00029968232932923495], "gradient_norm": 0.4608633518218994, "seconds": 0.3535647019999715, "nonpadding_tokens_per_second": 173190.36559256117, "ranks": [{"compute_seconds": 0.3513156279999521, "nonpadding_tokens": 3136, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3504384340000115, "nonpadding_tokens": 4317, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3507633330000317, "nonpadding_tokens": 10285, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3480343720000292, "nonpadding_tokens": 11787, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3460686319999695, "nonpadding_tokens": 4120, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3496094910000238, "nonpadding_tokens": 7395, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35118569300004765, "nonpadding_tokens": 15254, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3469423970000207, "nonpadding_tokens": 4940, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038973100004113803, "prefetch": {"data_wait_seconds": 0.010480826000573984, "consumed_batches": 1114, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1664, "tokens": 128898682, "loss_tokens": 130632, "loss": 2.3381613042746037, "lr": 0.00029968156611718193, "optimizer_lrs": [0.019978771074478797, 0.00029968156611718193], "gradient_norm": 0.38221824169158936, "seconds": 0.37158961300008286, "nonpadding_tokens_per_second": 351549.11609429424, "ranks": [{"compute_seconds": 0.3681694619999689, "nonpadding_tokens": 24883, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36533952800004954, "nonpadding_tokens": 8954, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3652387380000164, "nonpadding_tokens": 29040, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3664426990000038, "nonpadding_tokens": 24916, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3691999970000097, "nonpadding_tokens": 10881, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36716338600001563, "nonpadding_tokens": 19956, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3645278839999264, "nonpadding_tokens": 6880, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3652280609999252, "nonpadding_tokens": 5122, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038917799997761904, "prefetch": {"data_wait_seconds": 0.010490308000569826, "consumed_batches": 1115, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1665, "tokens": 128957357, "loss_tokens": 58675, "loss": 2.4442929804005114, "lr": 0.00029968122301325, "optimizer_lrs": [0.019978748200883335, 0.00029968122301325], "gradient_norm": 0.4081241190433502, "seconds": 0.2521947640000235, "nonpadding_tokens_per_second": 232657.48689371892, "ranks": [{"compute_seconds": 0.2466716159999578, "nonpadding_tokens": 10414, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2477627629999688, "nonpadding_tokens": 10067, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24760736899997937, "nonpadding_tokens": 2012, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24526827099998627, "nonpadding_tokens": 3273, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24901865300000736, "nonpadding_tokens": 8009, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2465873209999927, "nonpadding_tokens": 9848, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24374747199999547, "nonpadding_tokens": 11772, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24965027200005352, "nonpadding_tokens": 3280, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038594499994815124, "prefetch": {"data_wait_seconds": 0.010499575000565642, "consumed_batches": 1116, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1666, "tokens": 128992488, "loss_tokens": 35131, "loss": 2.199649837394324, "lr": 0.00029968101749529226, "optimizer_lrs": [0.01997873449968615, 0.00029968101749529226], "gradient_norm": 0.4704795181751251, "seconds": 0.18080141699999785, "nonpadding_tokens_per_second": 194307.1054581415, "ranks": [{"compute_seconds": 0.17758551400004308, "nonpadding_tokens": 6352, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.17811881400007223, "nonpadding_tokens": 4495, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.17519548500001747, "nonpadding_tokens": 2302, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.17574660100001438, "nonpadding_tokens": 3704, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1781690459999936, "nonpadding_tokens": 1988, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.17583617800005413, "nonpadding_tokens": 7086, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.1785232280000173, "nonpadding_tokens": 4766, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.17457542299996476, "nonpadding_tokens": 4438, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003412600000274324, "prefetch": {"data_wait_seconds": 0.0105082920005799, "consumed_batches": 1117, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1667, "tokens": 129066403, "loss_tokens": 73915, "loss": 2.4200595701143204, "lr": 0.00029968058487315666, "optimizer_lrs": [0.019978705658210447, 0.00029968058487315666], "gradient_norm": 0.384721577167511, "seconds": 0.27774285899999995, "nonpadding_tokens_per_second": 266127.45424356713, "ranks": [{"compute_seconds": 0.27284788800000115, "nonpadding_tokens": 3412, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27263384699995186, "nonpadding_tokens": 13763, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2730746029999409, "nonpadding_tokens": 7872, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2699229900000546, "nonpadding_tokens": 10385, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27539474599996083, "nonpadding_tokens": 4531, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2711736879999762, "nonpadding_tokens": 8290, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2685237650000545, "nonpadding_tokens": 9268, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2685224059999882, "nonpadding_tokens": 16394, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003543760000184193, "prefetch": {"data_wait_seconds": 0.010517301000575685, "consumed_batches": 1118, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1668, "tokens": 129164628, "loss_tokens": 98225, "loss": 2.557023097480275, "lr": 0.0002996800095122402, "optimizer_lrs": [0.019978667300816016, 0.0002996800095122402], "gradient_norm": 0.38306692242622375, "seconds": 0.32701199500002076, "nonpadding_tokens_per_second": 300371.2447917813, "ranks": [{"compute_seconds": 0.3232366910000337, "nonpadding_tokens": 9846, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3213190510000459, "nonpadding_tokens": 15681, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.323209592000012, "nonpadding_tokens": 10175, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.318258607999951, "nonpadding_tokens": 11080, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3162216420000732, "nonpadding_tokens": 24039, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32123034900007497, "nonpadding_tokens": 10824, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32433682400005637, "nonpadding_tokens": 10142, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31737102399995365, "nonpadding_tokens": 6438, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004185679999864078, "prefetch": {"data_wait_seconds": 0.010527299000571588, "consumed_batches": 1119, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1669, "tokens": 129214819, "loss_tokens": 50191, "loss": 2.6822758935864996, "lr": 0.0002996797153147528, "optimizer_lrs": [0.01997864768765019, 0.0002996797153147528], "gradient_norm": 0.42169108986854553, "seconds": 0.3103560539999535, "nonpadding_tokens_per_second": 161720.7054707801, "ranks": [{"compute_seconds": 0.30465672200000427, "nonpadding_tokens": 4682, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30414292399996157, "nonpadding_tokens": 3530, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30598038800008, "nonpadding_tokens": 3786, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3015476230000331, "nonpadding_tokens": 14233, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30748032499991496, "nonpadding_tokens": 8727, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3029666569999563, "nonpadding_tokens": 1855, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3071599620000143, "nonpadding_tokens": 11255, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3077800749999824, "nonpadding_tokens": 2123, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040422600000056264, "prefetch": {"data_wait_seconds": 0.010536907000599172, "consumed_batches": 1120, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1670, "tokens": 129251869, "loss_tokens": 37050, "loss": 2.8496160172064777, "lr": 0.0002996794980573861, "optimizer_lrs": [0.01997863320382574, 0.0002996794980573861], "gradient_norm": 0.4968695342540741, "seconds": 0.2901657700000442, "nonpadding_tokens_per_second": 127685.63294007546, "ranks": [{"compute_seconds": 0.25220017699996333, "nonpadding_tokens": 4285, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26053171299997757, "nonpadding_tokens": 3072, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2579345159999775, "nonpadding_tokens": 11346, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2583808519999593, "nonpadding_tokens": 5870, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2568225339999799, "nonpadding_tokens": 3559, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2582719230000521, "nonpadding_tokens": 2644, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26052852200007237, "nonpadding_tokens": 4230, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26172197799996866, "nonpadding_tokens": 2044, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0020052260000511524, "prefetch": {"data_wait_seconds": 0.010547544000544917, "consumed_batches": 1121, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1671, "tokens": 129312677, "loss_tokens": 60808, "loss": 2.718707859163268, "lr": 0.00029967914132609895, "optimizer_lrs": [0.019978609421739933, 0.00029967914132609895], "gradient_norm": 0.4227835536003113, "seconds": 0.3188458800000262, "nonpadding_tokens_per_second": 190712.82965925417, "ranks": [{"compute_seconds": 0.3143175140000949, "nonpadding_tokens": 14653, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31618648999995, "nonpadding_tokens": 13171, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3127444120000291, "nonpadding_tokens": 8378, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3159187999999631, "nonpadding_tokens": 8924, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31649980500003494, "nonpadding_tokens": 4956, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31492031599998427, "nonpadding_tokens": 2907, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3120526890000974, "nonpadding_tokens": 2445, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3122794299999896, "nonpadding_tokens": 5374, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042848600003253523, "prefetch": {"data_wait_seconds": 0.010557084000609507, "consumed_batches": 1122, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1672, "tokens": 129386192, "loss_tokens": 73515, "loss": 2.6147238233693804, "lr": 0.00029967870978432577, "optimizer_lrs": [0.019978580652288386, 0.00029967870978432577], "gradient_norm": 0.40614038705825806, "seconds": 0.23591095900007986, "nonpadding_tokens_per_second": 311621.80982009874, "ranks": [{"compute_seconds": 0.23155181900006028, "nonpadding_tokens": 3020, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23318543900006716, "nonpadding_tokens": 8456, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2305738240000892, "nonpadding_tokens": 2910, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23268165199999657, "nonpadding_tokens": 10114, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2290876089999756, "nonpadding_tokens": 4741, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23071348299993133, "nonpadding_tokens": 13554, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23337846600009016, "nonpadding_tokens": 20345, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23362067000005027, "nonpadding_tokens": 10375, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003888480000568961, "prefetch": {"data_wait_seconds": 0.010567437000602808, "consumed_batches": 1123, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1673, "tokens": 129462546, "loss_tokens": 76354, "loss": 2.393861773777405, "lr": 0.0002996782612706282, "optimizer_lrs": [0.019978550751375213, 0.0002996782612706282], "gradient_norm": 0.5263953804969788, "seconds": 0.24901197299993783, "nonpadding_tokens_per_second": 306627.82628536125, "ranks": [{"compute_seconds": 0.24390427499997713, "nonpadding_tokens": 3888, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24199598899997454, "nonpadding_tokens": 12838, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2421621259999256, "nonpadding_tokens": 11714, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24286004199996114, "nonpadding_tokens": 11257, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24122364499999094, "nonpadding_tokens": 14371, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24291234200006784, "nonpadding_tokens": 2983, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24103506699998434, "nonpadding_tokens": 9083, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24663954899995133, "nonpadding_tokens": 10220, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033333999999740627, "prefetch": {"data_wait_seconds": 0.010575504000669866, "consumed_batches": 1124, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1674, "tokens": 129524548, "loss_tokens": 62002, "loss": 2.372276297538789, "lr": 0.0002996778968326291, "optimizer_lrs": [0.019978526455508606, 0.0002996778968326291], "gradient_norm": 0.413492888212204, "seconds": 0.26311802300006093, "nonpadding_tokens_per_second": 235643.30292944488, "ranks": [{"compute_seconds": 0.26089036999997006, "nonpadding_tokens": 2985, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2569038780000028, "nonpadding_tokens": 4773, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2570824970000558, "nonpadding_tokens": 15710, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2601534760000277, "nonpadding_tokens": 3017, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2566137669999762, "nonpadding_tokens": 3866, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25900189900005444, "nonpadding_tokens": 11356, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2563645100000258, "nonpadding_tokens": 14998, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25738692300001276, "nonpadding_tokens": 5297, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035921599999255704, "prefetch": {"data_wait_seconds": 0.010584176000591583, "consumed_batches": 1125, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1675, "tokens": 129579137, "loss_tokens": 54589, "loss": 2.3357699124365716, "lr": 0.0002996775757965173, "optimizer_lrs": [0.019978505053101153, 0.0002996775757965173], "gradient_norm": 0.3807041645050049, "seconds": 0.2641654469999821, "nonpadding_tokens_per_second": 206647.0108787683, "ranks": [{"compute_seconds": 0.25117614200007665, "nonpadding_tokens": 2131, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2564574159999893, "nonpadding_tokens": 1458, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25353156400001353, "nonpadding_tokens": 4323, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2544100919999437, "nonpadding_tokens": 9038, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25270582999996805, "nonpadding_tokens": 10753, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2539978060000294, "nonpadding_tokens": 6188, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25659494199999244, "nonpadding_tokens": 11943, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25317309199999727, "nonpadding_tokens": 8755, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035312800002884615, "prefetch": {"data_wait_seconds": 0.010592776000592607, "consumed_batches": 1126, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1676, "tokens": 129626046, "loss_tokens": 46909, "loss": 2.6961384009465132, "lr": 0.0002996772997986416, "optimizer_lrs": [0.019978486653242776, 0.0002996772997986416], "gradient_norm": 0.5022971034049988, "seconds": 0.30960496900002, "nonpadding_tokens_per_second": 151512.42614583802, "ranks": [{"compute_seconds": 0.29726803799997015, "nonpadding_tokens": 4763, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.298989980999977, "nonpadding_tokens": 10099, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30263498599993, "nonpadding_tokens": 5401, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29717421899999863, "nonpadding_tokens": 6948, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29520926999998665, "nonpadding_tokens": 7210, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3007576219999919, "nonpadding_tokens": 2751, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29507141700003103, "nonpadding_tokens": 6349, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2957242260000612, "nonpadding_tokens": 3388, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0027882389999831503, "prefetch": {"data_wait_seconds": 0.01060186800062013, "consumed_batches": 1127, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1677, "tokens": 129674517, "loss_tokens": 48471, "loss": 2.5734076174413567, "lr": 0.00029967701448655436, "optimizer_lrs": [0.019978467632436957, 0.00029967701448655436], "gradient_norm": 0.44325655698776245, "seconds": 0.29649971600008485, "nonpadding_tokens_per_second": 163477.39098672906, "ranks": [{"compute_seconds": 0.2584551660000898, "nonpadding_tokens": 7493, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2575849449999623, "nonpadding_tokens": 9787, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2603093320000198, "nonpadding_tokens": 1553, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2583928010000136, "nonpadding_tokens": 3599, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2569023239999524, "nonpadding_tokens": 8654, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2597277930000246, "nonpadding_tokens": 3725, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2569895670000051, "nonpadding_tokens": 11040, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25756560499996795, "nonpadding_tokens": 2620, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.004723832999957267, "prefetch": {"data_wait_seconds": 0.010611025000642371, "consumed_batches": 1128, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1678, "tokens": 129717660, "loss_tokens": 43143, "loss": 2.56488197679809, "lr": 0.0002996767604304504, "optimizer_lrs": [0.019978450695363362, 0.0002996767604304504], "gradient_norm": 0.4137769341468811, "seconds": 0.18168796300005852, "nonpadding_tokens_per_second": 237456.56722446773, "ranks": [{"compute_seconds": 0.1787154809999265, "nonpadding_tokens": 3676, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.1781578029999764, "nonpadding_tokens": 6956, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.17530587099997774, "nonpadding_tokens": 2496, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.1760782180000433, "nonpadding_tokens": 8440, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.17435982499989677, "nonpadding_tokens": 2410, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.17724943900009293, "nonpadding_tokens": 2681, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.1782271079999873, "nonpadding_tokens": 9955, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.17938101499998993, "nonpadding_tokens": 6529, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003562760000477283, "prefetch": {"data_wait_seconds": 0.010618785000659159, "consumed_batches": 1129, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1679, "tokens": 129777921, "loss_tokens": 60261, "loss": 2.8097001999634923, "lr": 0.00029967640540462667, "optimizer_lrs": [0.019978427026975113, 0.00029967640540462667], "gradient_norm": 0.4320796728134155, "seconds": 0.30897717499999544, "nonpadding_tokens_per_second": 195033.82410044008, "ranks": [{"compute_seconds": 0.3048939899999823, "nonpadding_tokens": 4801, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3067286090000607, "nonpadding_tokens": 9800, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30388414200001534, "nonpadding_tokens": 2834, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3040221249999604, "nonpadding_tokens": 11266, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3025314860000208, "nonpadding_tokens": 6269, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3051986749999287, "nonpadding_tokens": 10975, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30600238799991075, "nonpadding_tokens": 5025, "peak_allocated_bytes": 14988922368, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3064258189999691, "nonpadding_tokens": 9291, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003424640000275758, "prefetch": {"data_wait_seconds": 0.01062642500069444, "consumed_batches": 1130, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1680, "tokens": 129838587, "loss_tokens": 60666, "loss": 2.8291165974351364, "lr": 0.00029967604779616353, "optimizer_lrs": [0.019978403186410903, 0.00029967604779616353], "gradient_norm": 0.4113568067550659, "seconds": 0.2575013929999841, "nonpadding_tokens_per_second": 235594.84200539353, "ranks": [{"compute_seconds": 0.2545814910000672, "nonpadding_tokens": 5321, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2548818320000237, "nonpadding_tokens": 3720, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25171933000001445, "nonpadding_tokens": 5704, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25265264999995907, "nonpadding_tokens": 3693, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2503809669999555, "nonpadding_tokens": 6577, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2524829609999415, "nonpadding_tokens": 11950, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2507055720000153, "nonpadding_tokens": 12483, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2552321140000231, "nonpadding_tokens": 11218, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003390959999478582, "prefetch": {"data_wait_seconds": 0.010634706000772098, "consumed_batches": 1131, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1681, "tokens": 129904802, "loss_tokens": 66215, "loss": 2.550251312013894, "lr": 0.000299675657252815, "optimizer_lrs": [0.01997837715018767, 0.000299675657252815], "gradient_norm": 0.357163667678833, "seconds": 0.32868115799999487, "nonpadding_tokens_per_second": 201456.63476091635, "ranks": [{"compute_seconds": 0.32551838200004113, "nonpadding_tokens": 11975, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.326238887000045, "nonpadding_tokens": 2262, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32590196499995727, "nonpadding_tokens": 14220, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32399322399999164, "nonpadding_tokens": 6733, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32237624499998674, "nonpadding_tokens": 9271, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3254183550000107, "nonpadding_tokens": 3499, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3264486030000171, "nonpadding_tokens": 15354, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3230891650000558, "nonpadding_tokens": 2901, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035007600001790706, "prefetch": {"data_wait_seconds": 0.010642861000704329, "consumed_batches": 1132, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1682, "tokens": 129939924, "loss_tokens": 35122, "loss": 2.5544196835316897, "lr": 0.00029967545000402044, "optimizer_lrs": [0.019978363333601364, 0.00029967545000402044], "gradient_norm": 0.3881479501724243, "seconds": 0.25311877199999344, "nonpadding_tokens_per_second": 138756.99428567436, "ranks": [{"compute_seconds": 0.24638849600000867, "nonpadding_tokens": 3447, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24625397799991333, "nonpadding_tokens": 5920, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24848743400002604, "nonpadding_tokens": 5848, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24935775400001603, "nonpadding_tokens": 4161, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24177626799996688, "nonpadding_tokens": 3338, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2449872959999766, "nonpadding_tokens": 2451, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24955545399996026, "nonpadding_tokens": 4384, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2507736950000208, "nonpadding_tokens": 5573, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003680460000623498, "prefetch": {"data_wait_seconds": 0.01065141100066569, "consumed_batches": 1133, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1683, "tokens": 130010989, "loss_tokens": 71065, "loss": 2.8248839090973052, "lr": 0.0002996750304596324, "optimizer_lrs": [0.019978335363975492, 0.0002996750304596324], "gradient_norm": 0.3651837408542633, "seconds": 0.3312813909999477, "nonpadding_tokens_per_second": 214515.52043263192, "ranks": [{"compute_seconds": 0.32811934099993323, "nonpadding_tokens": 14785, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.324881481000034, "nonpadding_tokens": 5783, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3253792140000087, "nonpadding_tokens": 1604, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.326043559000027, "nonpadding_tokens": 19658, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32450754900003176, "nonpadding_tokens": 6906, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3258097359999965, "nonpadding_tokens": 14823, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.328406804999986, "nonpadding_tokens": 4591, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32890310100003717, "nonpadding_tokens": 2915, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003790580000213595, "prefetch": {"data_wait_seconds": 0.010660757000735543, "consumed_batches": 1134, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1684, "tokens": 130070077, "loss_tokens": 59088, "loss": 2.6456040672386947, "lr": 0.00029967468141742584, "optimizer_lrs": [0.019978312094495056, 0.00029967468141742584], "gradient_norm": 0.42766475677490234, "seconds": 0.447507824000013, "nonpadding_tokens_per_second": 132037.91493933363, "ranks": [{"compute_seconds": 0.44516381300002195, "nonpadding_tokens": 5282, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4412216800000124, "nonpadding_tokens": 8396, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4438994099999718, "nonpadding_tokens": 3906, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.44463186799998766, "nonpadding_tokens": 10358, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4408073990000503, "nonpadding_tokens": 2201, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4434797039999694, "nonpadding_tokens": 6154, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4444130340000356, "nonpadding_tokens": 14530, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4452212270000473, "nonpadding_tokens": 8261, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034535999998297484, "prefetch": {"data_wait_seconds": 0.01066971700068109, "consumed_batches": 1135, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1685, "tokens": 130140438, "loss_tokens": 70361, "loss": 2.580645705717656, "lr": 0.0002996742655397393, "optimizer_lrs": [0.019978284369315952, 0.0002996742655397393], "gradient_norm": 0.36685243248939514, "seconds": 0.2235800340000651, "nonpadding_tokens_per_second": 314701.62492228404, "ranks": [{"compute_seconds": 0.20238213499999347, "nonpadding_tokens": 9358, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20757565200005956, "nonpadding_tokens": 12438, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20649858900003437, "nonpadding_tokens": 9699, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2050997069999312, "nonpadding_tokens": 10741, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20786675299996205, "nonpadding_tokens": 12655, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20474809899997126, "nonpadding_tokens": 4830, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2033985619999612, "nonpadding_tokens": 6410, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20411210100007793, "nonpadding_tokens": 4230, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0035874960000228384, "prefetch": {"data_wait_seconds": 0.010678678000729747, "consumed_batches": 1136, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1686, "tokens": 130198474, "loss_tokens": 58036, "loss": 2.3575528551244056, "lr": 0.00029967392231085483, "optimizer_lrs": [0.019978261487390323, 0.00029967392231085483], "gradient_norm": 0.43717485666275024, "seconds": 0.32511458599992693, "nonpadding_tokens_per_second": 178509.37023174053, "ranks": [{"compute_seconds": 0.3191149830000768, "nonpadding_tokens": 2586, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31909682899993186, "nonpadding_tokens": 1638, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3216268730000138, "nonpadding_tokens": 8017, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3202240130000291, "nonpadding_tokens": 9608, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.322749425999973, "nonpadding_tokens": 4936, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32144455499997093, "nonpadding_tokens": 9831, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32277683600000273, "nonpadding_tokens": 16007, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31910638099998323, "nonpadding_tokens": 5413, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040750800008027, "prefetch": {"data_wait_seconds": 0.010688227000699158, "consumed_batches": 1137, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1687, "tokens": 130268303, "loss_tokens": 69829, "loss": 2.7935201528018445, "lr": 0.000299673509098093, "optimizer_lrs": [0.019978233939872865, 0.000299673509098093], "gradient_norm": 0.4035918414592743, "seconds": 0.2831824449999658, "nonpadding_tokens_per_second": 246586.61309322488, "ranks": [{"compute_seconds": 0.2413798110000016, "nonpadding_tokens": 10283, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2464815749999616, "nonpadding_tokens": 10204, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2465793119999944, "nonpadding_tokens": 1908, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24939375399992514, "nonpadding_tokens": 3088, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2499030850000281, "nonpadding_tokens": 12114, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24858480199998212, "nonpadding_tokens": 10272, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2495178770000166, "nonpadding_tokens": 16732, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25037087800001245, "nonpadding_tokens": 5228, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.02300566099995649, "prefetch": {"data_wait_seconds": 0.010697011000686985, "consumed_batches": 1138, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1688, "tokens": 130343011, "loss_tokens": 74708, "loss": 2.4736729165551212, "lr": 0.0002996730667245122, "optimizer_lrs": [0.019978204448300813, 0.0002996730667245122], "gradient_norm": 0.40333861112594604, "seconds": 0.3200883120000526, "nonpadding_tokens_per_second": 233398.08796263614, "ranks": [{"compute_seconds": 0.3159185660000503, "nonpadding_tokens": 8897, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31594328599999244, "nonpadding_tokens": 7770, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3127851949999467, "nonpadding_tokens": 3741, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3170924769999601, "nonpadding_tokens": 11274, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31161692899991067, "nonpadding_tokens": 10622, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31440727000006063, "nonpadding_tokens": 14071, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3171861880000506, "nonpadding_tokens": 13786, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3177193650000163, "nonpadding_tokens": 4547, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003323680000448803, "prefetch": {"data_wait_seconds": 0.010705363000695911, "consumed_batches": 1139, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1689, "tokens": 130387366, "loss_tokens": 44355, "loss": 2.2364475890542215, "lr": 0.00029967280394070466, "optimizer_lrs": [0.01997818692938031, 0.00029967280394070466], "gradient_norm": 0.5357401371002197, "seconds": 0.25728136399993673, "nonpadding_tokens_per_second": 172398.80615686922, "ranks": [{"compute_seconds": 0.25401280099993073, "nonpadding_tokens": 2190, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25120110800003204, "nonpadding_tokens": 3708, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2513904590000493, "nonpadding_tokens": 5643, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2541429099999277, "nonpadding_tokens": 3760, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2503971730000103, "nonpadding_tokens": 9563, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.253327626999976, "nonpadding_tokens": 1754, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25431599900002766, "nonpadding_tokens": 10620, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.254931430000056, "nonpadding_tokens": 7117, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041681199991217, "prefetch": {"data_wait_seconds": 0.010715267000776407, "consumed_batches": 1140, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1690, "tokens": 130495459, "loss_tokens": 108093, "loss": 2.828788635711841, "lr": 0.00029967216309578494, "optimizer_lrs": [0.019978144206385665, 0.00029967216309578494], "gradient_norm": 0.42617857456207275, "seconds": 0.31373088899999857, "nonpadding_tokens_per_second": 344540.50841006124, "ranks": [{"compute_seconds": 0.27857034199996633, "nonpadding_tokens": 1720, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2803928509999878, "nonpadding_tokens": 4469, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2794416230000252, "nonpadding_tokens": 2647, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2798909560000311, "nonpadding_tokens": 18574, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2759617539999226, "nonpadding_tokens": 27222, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.279200746000015, "nonpadding_tokens": 11184, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28015570299999126, "nonpadding_tokens": 19793, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27710864199991647, "nonpadding_tokens": 22484, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.019988604999980453, "prefetch": {"data_wait_seconds": 0.01072461500082511, "consumed_batches": 1141, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1691, "tokens": 130547781, "loss_tokens": 52322, "loss": 2.6620992961851613, "lr": 0.00029967185267235666, "optimizer_lrs": [0.019978123511490444, 0.00029967185267235666], "gradient_norm": 0.47685155272483826, "seconds": 0.254662991000032, "nonpadding_tokens_per_second": 205455.84497589376, "ranks": [{"compute_seconds": 0.25148537400002624, "nonpadding_tokens": 7412, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2481020369999669, "nonpadding_tokens": 6925, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2487408910000113, "nonpadding_tokens": 8137, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24939352700005202, "nonpadding_tokens": 4116, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2520019820000243, "nonpadding_tokens": 4995, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25039936999996826, "nonpadding_tokens": 2881, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24771176299998388, "nonpadding_tokens": 11693, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25227813700007573, "nonpadding_tokens": 6163, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003633860000036293, "prefetch": {"data_wait_seconds": 0.010733628000778594, "consumed_batches": 1142, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1692, "tokens": 130619386, "loss_tokens": 71605, "loss": 2.6825049315690244, "lr": 0.0002996714276062023, "optimizer_lrs": [0.01997809517374682, 0.0002996714276062023], "gradient_norm": 0.7540590167045593, "seconds": 0.2894210090000797, "nonpadding_tokens_per_second": 247407.74779062523, "ranks": [{"compute_seconds": 0.2869979110000713, "nonpadding_tokens": 9391, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28688739599999735, "nonpadding_tokens": 9612, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2839812400000028, "nonpadding_tokens": 3428, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28514496999991934, "nonpadding_tokens": 7721, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2855138200000056, "nonpadding_tokens": 8175, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2856845160000603, "nonpadding_tokens": 12513, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28274554400002216, "nonpadding_tokens": 5177, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28347755699996924, "nonpadding_tokens": 15588, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003983640000342348, "prefetch": {"data_wait_seconds": 0.010743237000795602, "consumed_batches": 1143, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1693, "tokens": 130683807, "loss_tokens": 64421, "loss": 2.6158453765076604, "lr": 0.000299671044951345, "optimizer_lrs": [0.019978069663423, 0.000299671044951345], "gradient_norm": 0.37986481189727783, "seconds": 0.29670002900002146, "nonpadding_tokens_per_second": 217125.02090788586, "ranks": [{"compute_seconds": 0.2916459500000883, "nonpadding_tokens": 15768, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29370505800000046, "nonpadding_tokens": 8214, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.292815217999987, "nonpadding_tokens": 4815, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29354444099999455, "nonpadding_tokens": 13343, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29368675199998506, "nonpadding_tokens": 6370, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2912790560000076, "nonpadding_tokens": 2085, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2937586990000227, "nonpadding_tokens": 6484, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2943644619999759, "nonpadding_tokens": 7342, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004040129999793862, "prefetch": {"data_wait_seconds": 0.010752136000817814, "consumed_batches": 1144, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1694, "tokens": 130783183, "loss_tokens": 99376, "loss": 2.230349254347126, "lr": 0.000299670454230793, "optimizer_lrs": [0.019978030282052868, 0.000299670454230793], "gradient_norm": 0.47132086753845215, "seconds": 0.26394407400005093, "nonpadding_tokens_per_second": 376504.00137409725, "ranks": [{"compute_seconds": 0.2599816980000469, "nonpadding_tokens": 13034, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26162675200009744, "nonpadding_tokens": 4971, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25845663600000535, "nonpadding_tokens": 17540, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2609978329999194, "nonpadding_tokens": 9928, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25726741900007255, "nonpadding_tokens": 13721, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2601453639999818, "nonpadding_tokens": 10518, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25719936799998777, "nonpadding_tokens": 10493, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2616680769999675, "nonpadding_tokens": 19171, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033603200006382394, "prefetch": {"data_wait_seconds": 0.010761524000784561, "consumed_batches": 1145, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1695, "tokens": 130835328, "loss_tokens": 52145, "loss": 2.510711417681465, "lr": 0.0002996701440536534, "optimizer_lrs": [0.01997800960357689, 0.0002996701440536534], "gradient_norm": 0.39005187153816223, "seconds": 0.43460625000000164, "nonpadding_tokens_per_second": 119982.16776679996, "ranks": [{"compute_seconds": 0.4315984050000452, "nonpadding_tokens": 4597, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4284859749999441, "nonpadding_tokens": 2787, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4284247040000082, "nonpadding_tokens": 6811, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4320476430000326, "nonpadding_tokens": 4411, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4275197509999771, "nonpadding_tokens": 9996, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4295607530000325, "nonpadding_tokens": 12037, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.43227303399999073, "nonpadding_tokens": 3018, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.42831151699999737, "nonpadding_tokens": 8488, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037028600002031453, "prefetch": {"data_wait_seconds": 0.010769931000780275, "consumed_batches": 1146, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1696, "tokens": 130912843, "loss_tokens": 77515, "loss": 2.2907362687866866, "lr": 0.00029966968269734225, "optimizer_lrs": [0.019977978846489487, 0.00029966968269734225], "gradient_norm": 0.5651464462280273, "seconds": 0.3206672130000925, "nonpadding_tokens_per_second": 241730.3573844877, "ranks": [{"compute_seconds": 0.31330183000000034, "nonpadding_tokens": 2979, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3152405490000092, "nonpadding_tokens": 5118, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3119932700000163, "nonpadding_tokens": 11863, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3125254369999766, "nonpadding_tokens": 7052, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3108928949999381, "nonpadding_tokens": 17947, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.314536313000076, "nonpadding_tokens": 7343, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3169183139999632, "nonpadding_tokens": 19347, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3181493690000252, "nonpadding_tokens": 5866, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035616000002391957, "prefetch": {"data_wait_seconds": 0.010779271000799895, "consumed_batches": 1147, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1697, "tokens": 130978160, "loss_tokens": 65317, "loss": 2.2681896558323253, "lr": 0.00029966929369149047, "optimizer_lrs": [0.019977952912766033, 0.00029966929369149047], "gradient_norm": 0.3580934703350067, "seconds": 0.2900222329999451, "nonpadding_tokens_per_second": 225213.76835276058, "ranks": [{"compute_seconds": 0.28684110100005, "nonpadding_tokens": 10463, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2875361320000138, "nonpadding_tokens": 23311, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2843249719999221, "nonpadding_tokens": 4749, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2845630720000827, "nonpadding_tokens": 5974, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2834477280000556, "nonpadding_tokens": 9736, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2849194959999295, "nonpadding_tokens": 1531, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2834512869999344, "nonpadding_tokens": 5703, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2841360079999049, "nonpadding_tokens": 3850, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003485710000177278, "prefetch": {"data_wait_seconds": 0.010787358000769132, "consumed_batches": 1148, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1698, "tokens": 131026950, "loss_tokens": 48790, "loss": 2.863869389219102, "lr": 0.0002996690029656493, "optimizer_lrs": [0.019977933531043286, 0.0002996690029656493], "gradient_norm": 0.3931620121002197, "seconds": 0.2433960719999959, "nonpadding_tokens_per_second": 200455.16593217995, "ranks": [{"compute_seconds": 0.23990786500007744, "nonpadding_tokens": 6184, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2377922220000528, "nonpadding_tokens": 7486, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23773818300003313, "nonpadding_tokens": 4760, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24043915599997945, "nonpadding_tokens": 5031, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24104907100002038, "nonpadding_tokens": 10130, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2396355799999128, "nonpadding_tokens": 2810, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23661372399999436, "nonpadding_tokens": 10290, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23763718700001846, "nonpadding_tokens": 2099, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039537299994663044, "prefetch": {"data_wait_seconds": 0.010796145000725232, "consumed_batches": 1149, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1699, "tokens": 131088948, "loss_tokens": 61998, "loss": 2.9087190816639246, "lr": 0.0002996686333530078, "optimizer_lrs": [0.01997790889020052, 0.0002996686333530078], "gradient_norm": 0.3730836808681488, "seconds": 0.26158851000002414, "nonpadding_tokens_per_second": 237005.82261810458, "ranks": [{"compute_seconds": 0.2581062900000006, "nonpadding_tokens": 2506, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25593164300005355, "nonpadding_tokens": 5546, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25642141900004845, "nonpadding_tokens": 6292, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25931297700003597, "nonpadding_tokens": 4810, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25587110800006485, "nonpadding_tokens": 12593, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25713391199997204, "nonpadding_tokens": 3786, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2554875399999901, "nonpadding_tokens": 6006, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2562296510000124, "nonpadding_tokens": 20459, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003831820000641528, "prefetch": {"data_wait_seconds": 0.010804484000644266, "consumed_batches": 1150, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1700, "tokens": 131163139, "loss_tokens": 74191, "loss": 2.387137169602782, "lr": 0.0002996681907787711, "optimizer_lrs": [0.019977879385251408, 0.0002996681907787711], "gradient_norm": 0.4360562264919281, "seconds": 0.29009640699996453, "nonpadding_tokens_per_second": 255746.01480675722, "ranks": [{"compute_seconds": 0.2879370939999717, "nonpadding_tokens": 4305, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28400659900000846, "nonpadding_tokens": 19086, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.284418708999965, "nonpadding_tokens": 4343, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2849157020000348, "nonpadding_tokens": 9568, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2836927509999896, "nonpadding_tokens": 9676, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28496041799996874, "nonpadding_tokens": 11780, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28735902900007204, "nonpadding_tokens": 5410, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2841941740000493, "nonpadding_tokens": 10023, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037384999995992985, "prefetch": {"data_wait_seconds": 0.010812592000618793, "consumed_batches": 1151, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1701, "tokens": 131235767, "loss_tokens": 72628, "loss": 2.29042797715757, "lr": 0.00029966775724262005, "optimizer_lrs": [0.019977850482841338, 0.00029966775724262005], "gradient_norm": 0.4059711694717407, "seconds": 0.29531506299997545, "nonpadding_tokens_per_second": 245933.95020966485, "ranks": [{"compute_seconds": 0.2929321420000406, "nonpadding_tokens": 12632, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2883218939999779, "nonpadding_tokens": 23240, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28936551899994356, "nonpadding_tokens": 2496, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2905531029999793, "nonpadding_tokens": 2330, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28906861600000866, "nonpadding_tokens": 5265, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29144122299999253, "nonpadding_tokens": 4376, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2921187939999754, "nonpadding_tokens": 5667, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2926159580000558, "nonpadding_tokens": 16622, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003966289999652872, "prefetch": {"data_wait_seconds": 0.010819858000630234, "consumed_batches": 1152, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1702, "tokens": 131290483, "loss_tokens": 54716, "loss": 2.0712698582224576, "lr": 0.00029966743044132553, "optimizer_lrs": [0.01997782869608837, 0.00029966743044132553], "gradient_norm": 0.40231844782829285, "seconds": 0.3127303539999957, "nonpadding_tokens_per_second": 174962.22960180178, "ranks": [{"compute_seconds": 0.3031662559999404, "nonpadding_tokens": 14743, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3080181210000319, "nonpadding_tokens": 5145, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30521334299999126, "nonpadding_tokens": 11814, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3059288639999522, "nonpadding_tokens": 3050, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30441759699999693, "nonpadding_tokens": 4723, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3038610480000443, "nonpadding_tokens": 2666, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3067791660000694, "nonpadding_tokens": 10839, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3066108829999621, "nonpadding_tokens": 1736, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038319899999805784, "prefetch": {"data_wait_seconds": 0.010829134000644558, "consumed_batches": 1153, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1703, "tokens": 131343429, "loss_tokens": 52946, "loss": 2.398890349365391, "lr": 0.00029966711405893324, "optimizer_lrs": [0.019977807603928883, 0.00029966711405893324], "gradient_norm": 1.0110368728637695, "seconds": 0.313795425999956, "nonpadding_tokens_per_second": 168727.7621440136, "ranks": [{"compute_seconds": 0.3085755800000243, "nonpadding_tokens": 2784, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3105692840000529, "nonpadding_tokens": 10992, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30762675200003287, "nonpadding_tokens": 9737, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.307941851999999, "nonpadding_tokens": 9855, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31143390399995496, "nonpadding_tokens": 5402, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3085920510000051, "nonpadding_tokens": 5223, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3110107600000447, "nonpadding_tokens": 2413, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3067825599999878, "nonpadding_tokens": 6540, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003495949999887671, "prefetch": {"data_wait_seconds": 0.010837959000653541, "consumed_batches": 1154, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1704, "tokens": 131413169, "loss_tokens": 69740, "loss": 2.5151594314597077, "lr": 0.00029966669709359067, "optimizer_lrs": [0.01997777980623938, 0.00029966669709359067], "gradient_norm": 0.38945871591567993, "seconds": 0.24379301900000883, "nonpadding_tokens_per_second": 286062.333884948, "ranks": [{"compute_seconds": 0.239622762999943, "nonpadding_tokens": 6303, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23999334100005854, "nonpadding_tokens": 7870, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24077207800007727, "nonpadding_tokens": 5651, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24116292799999428, "nonpadding_tokens": 11608, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2415420949999998, "nonpadding_tokens": 6184, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23877532899996368, "nonpadding_tokens": 14555, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23665630400000737, "nonpadding_tokens": 13264, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23690399499992054, "nonpadding_tokens": 4305, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000344649999988178, "prefetch": {"data_wait_seconds": 0.010845921000694148, "consumed_batches": 1155, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1705, "tokens": 131454843, "loss_tokens": 41674, "loss": 2.534045133056582, "lr": 0.00029966644780637437, "optimizer_lrs": [0.019977763187091626, 0.00029966644780637437], "gradient_norm": 0.4965771734714508, "seconds": 0.28072292900003504, "nonpadding_tokens_per_second": 148452.42655613215, "ranks": [{"compute_seconds": 0.2784677900000361, "nonpadding_tokens": 12265, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27819350000004306, "nonpadding_tokens": 4975, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27556158699997013, "nonpadding_tokens": 2876, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2776604139999108, "nonpadding_tokens": 5548, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2784054500000366, "nonpadding_tokens": 5146, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27576552299990453, "nonpadding_tokens": 1731, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2747512280000137, "nonpadding_tokens": 3163, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27460534300007566, "nonpadding_tokens": 5970, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003831609999451757, "prefetch": {"data_wait_seconds": 0.010855699000742902, "consumed_batches": 1156, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1706, "tokens": 131510864, "loss_tokens": 56021, "loss": 2.6225544884953855, "lr": 0.0002996661125510402, "optimizer_lrs": [0.019977740836736015, 0.0002996661125510402], "gradient_norm": 0.42556965351104736, "seconds": 0.32147897700008343, "nonpadding_tokens_per_second": 174260.228531166, "ranks": [{"compute_seconds": 0.3192366119999406, "nonpadding_tokens": 13437, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31548578699994323, "nonpadding_tokens": 13791, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31827747299996645, "nonpadding_tokens": 3886, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3167092119999779, "nonpadding_tokens": 12145, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3152735879999682, "nonpadding_tokens": 2116, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3178631860000678, "nonpadding_tokens": 1585, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3190206939999598, "nonpadding_tokens": 1528, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3155551250000599, "nonpadding_tokens": 7533, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035824099995807046, "prefetch": {"data_wait_seconds": 0.010864471000743947, "consumed_batches": 1157, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1707, "tokens": 131587171, "loss_tokens": 76307, "loss": 2.284970579370176, "lr": 0.00029966565562441284, "optimizer_lrs": [0.019977710374960857, 0.00029966565562441284], "gradient_norm": 0.4674340784549713, "seconds": 0.37529497499997433, "nonpadding_tokens_per_second": 203325.39757561428, "ranks": [{"compute_seconds": 0.37287805500000104, "nonpadding_tokens": 10984, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36889455700008966, "nonpadding_tokens": 6308, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36937084800001685, "nonpadding_tokens": 18256, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.370168921999948, "nonpadding_tokens": 6713, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.37246466800002054, "nonpadding_tokens": 5568, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3710198339999806, "nonpadding_tokens": 6495, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.372170471000004, "nonpadding_tokens": 9488, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37288487499995426, "nonpadding_tokens": 12495, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003691199999593664, "prefetch": {"data_wait_seconds": 0.010874291000689595, "consumed_batches": 1158, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1708, "tokens": 131641512, "loss_tokens": 54341, "loss": 2.593868464879189, "lr": 0.00029966533004004473, "optimizer_lrs": [0.019977688669336316, 0.00029966533004004473], "gradient_norm": 0.4552605450153351, "seconds": 0.3114653369999587, "nonpadding_tokens_per_second": 174468.8526929313, "ranks": [{"compute_seconds": 0.3056639590000714, "nonpadding_tokens": 3457, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30866238200007956, "nonpadding_tokens": 11628, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30747415300004377, "nonpadding_tokens": 5292, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.306074573999922, "nonpadding_tokens": 5990, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30474504599999364, "nonpadding_tokens": 5586, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3067753730000504, "nonpadding_tokens": 2346, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30443751600000724, "nonpadding_tokens": 15335, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3048883569999816, "nonpadding_tokens": 4707, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034772399999383197, "prefetch": {"data_wait_seconds": 0.010882901000698553, "consumed_batches": 1159, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1709, "tokens": 131698953, "loss_tokens": 57441, "loss": 2.417318422381226, "lr": 0.0002996649857099589, "optimizer_lrs": [0.019977665713997263, 0.0002996649857099589], "gradient_norm": 0.4479213058948517, "seconds": 0.3341606519999232, "nonpadding_tokens_per_second": 171896.36079598387, "ranks": [{"compute_seconds": 0.3303797259999328, "nonpadding_tokens": 14404, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3313835559999916, "nonpadding_tokens": 1507, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3286536629999546, "nonpadding_tokens": 3746, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32932253599994965, "nonpadding_tokens": 2667, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3271293700000797, "nonpadding_tokens": 5961, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3289975889999823, "nonpadding_tokens": 6404, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3313420409999708, "nonpadding_tokens": 5126, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33185329200000524, "nonpadding_tokens": 17626, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003896489999988262, "prefetch": {"data_wait_seconds": 0.010892694000702363, "consumed_batches": 1160, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1710, "tokens": 131754880, "loss_tokens": 55927, "loss": 2.548283923686234, "lr": 0.00029966465028565247, "optimizer_lrs": [0.019977643352376832, 0.00029966465028565247], "gradient_norm": 0.4043577313423157, "seconds": 0.2663660839999693, "nonpadding_tokens_per_second": 209962.91705067994, "ranks": [{"compute_seconds": 0.2637967299999673, "nonpadding_tokens": 11658, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2641234710000617, "nonpadding_tokens": 3312, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.261301321000019, "nonpadding_tokens": 10000, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26387054299993906, "nonpadding_tokens": 9338, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25989190300003884, "nonpadding_tokens": 6769, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2613137150000284, "nonpadding_tokens": 4365, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2606165109999665, "nonpadding_tokens": 1791, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2613171919999786, "nonpadding_tokens": 8694, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036575499996160943, "prefetch": {"data_wait_seconds": 0.010902410000724672, "consumed_batches": 1161, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1711, "tokens": 131861363, "loss_tokens": 106483, "loss": 2.786910527971601, "lr": 0.00029966401118641464, "optimizer_lrs": [0.01997760074576098, 0.00029966401118641464], "gradient_norm": 5.667814254760742, "seconds": 0.3533450439999797, "nonpadding_tokens_per_second": 301356.99313786364, "ranks": [{"compute_seconds": 0.3452261449999696, "nonpadding_tokens": 10454, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3472311710000895, "nonpadding_tokens": 19205, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35026662799998576, "nonpadding_tokens": 16728, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3454079370000045, "nonpadding_tokens": 25437, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35092806699992707, "nonpadding_tokens": 14183, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34647200500000963, "nonpadding_tokens": 10834, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.343639549000045, "nonpadding_tokens": 1488, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3508726990000923, "nonpadding_tokens": 8154, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040007699999478064, "prefetch": {"data_wait_seconds": 0.010912152000699393, "consumed_batches": 1162, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1712, "tokens": 131918843, "loss_tokens": 57480, "loss": 2.811396898921364, "lr": 0.0002996636659452483, "optimizer_lrs": [0.019977577729683223, 0.0002996636659452483], "gradient_norm": 0.49607425928115845, "seconds": 0.378633949999994, "nonpadding_tokens_per_second": 151808.89088260816, "ranks": [{"compute_seconds": 0.3577897050000729, "nonpadding_tokens": 18289, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3762501850000035, "nonpadding_tokens": 10525, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37470384199991713, "nonpadding_tokens": 2917, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3751133780000373, "nonpadding_tokens": 3590, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3759847979999904, "nonpadding_tokens": 5442, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37287527000000864, "nonpadding_tokens": 2583, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3715718089999882, "nonpadding_tokens": 8179, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3758858580000606, "nonpadding_tokens": 5955, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003695420000440208, "prefetch": {"data_wait_seconds": 0.010921208000652882, "consumed_batches": 1163, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1713, "tokens": 132002739, "loss_tokens": 83896, "loss": 2.4087381996757893, "lr": 0.0002996631617242834, "optimizer_lrs": [0.019977544114952228, 0.0002996631617242834], "gradient_norm": 0.4087928831577301, "seconds": 0.31032077700001537, "nonpadding_tokens_per_second": 270352.5068835331, "ranks": [{"compute_seconds": 0.29126903500002754, "nonpadding_tokens": 18048, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29348276000007445, "nonpadding_tokens": 3379, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2959228519999897, "nonpadding_tokens": 6233, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29442400699997506, "nonpadding_tokens": 6655, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29302980399995704, "nonpadding_tokens": 17833, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29426859299996977, "nonpadding_tokens": 6478, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29729958899997655, "nonpadding_tokens": 12419, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29727866399991854, "nonpadding_tokens": 12851, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038531699999566627, "prefetch": {"data_wait_seconds": 0.010930367000653973, "consumed_batches": 1164, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1714, "tokens": 132056790, "loss_tokens": 54051, "loss": 2.6421429529518417, "lr": 0.0002996628366741136, "optimizer_lrs": [0.019977522444940908, 0.0002996628366741136], "gradient_norm": 0.4392615258693695, "seconds": 0.2785467889999609, "nonpadding_tokens_per_second": 194046.39412306272, "ranks": [{"compute_seconds": 0.27624935400001505, "nonpadding_tokens": 5721, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2758718799999542, "nonpadding_tokens": 2369, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2749785600000223, "nonpadding_tokens": 3620, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2736200210000561, "nonpadding_tokens": 4329, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2717646489999197, "nonpadding_tokens": 1495, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2728139549999469, "nonpadding_tokens": 17833, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27166081799998665, "nonpadding_tokens": 14971, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2724051169999484, "nonpadding_tokens": 3713, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034212000002753484, "prefetch": {"data_wait_seconds": 0.010938158000683984, "consumed_batches": 1165, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1715, "tokens": 132146943, "loss_tokens": 90153, "loss": 2.2377019830177587, "lr": 0.00029966229416654417, "optimizer_lrs": [0.019977486277769612, 0.00029966229416654417], "gradient_norm": 0.37348446249961853, "seconds": 0.29468860700001187, "nonpadding_tokens_per_second": 305926.3163166548, "ranks": [{"compute_seconds": 0.29243520700003955, "nonpadding_tokens": 8846, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29165223199993306, "nonpadding_tokens": 11103, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28817152700003135, "nonpadding_tokens": 4946, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28827990400009185, "nonpadding_tokens": 6616, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28708419200006574, "nonpadding_tokens": 2482, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28988451899999745, "nonpadding_tokens": 16293, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29098221600008856, "nonpadding_tokens": 19028, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2920529309999438, "nonpadding_tokens": 20839, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003517700000656987, "prefetch": {"data_wait_seconds": 0.010946745000708802, "consumed_batches": 1166, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1716, "tokens": 132225861, "loss_tokens": 78918, "loss": 2.5209695506728504, "lr": 0.0002996618189095476, "optimizer_lrs": [0.01997745459396984, 0.0002996618189095476], "gradient_norm": 0.5971899628639221, "seconds": 0.35564300400005777, "nonpadding_tokens_per_second": 221902.2984070486, "ranks": [{"compute_seconds": 0.35287583899992114, "nonpadding_tokens": 15923, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3534259599999814, "nonpadding_tokens": 2610, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35088051599996106, "nonpadding_tokens": 14892, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.35191967600007956, "nonpadding_tokens": 2602, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3503361149998909, "nonpadding_tokens": 9696, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3525377880000633, "nonpadding_tokens": 2989, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3517425380000532, "nonpadding_tokens": 12353, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3510660969999435, "nonpadding_tokens": 17853, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035499499995239603, "prefetch": {"data_wait_seconds": 0.01095525500063843, "consumed_batches": 1167, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1717, "tokens": 132304673, "loss_tokens": 78812, "loss": 2.6306213679388923, "lr": 0.0002996613439578097, "optimizer_lrs": [0.01997742293052065, 0.0002996613439578097], "gradient_norm": 0.735478401184082, "seconds": 0.44445169700009046, "nonpadding_tokens_per_second": 177324.10638086495, "ranks": [{"compute_seconds": 0.442186042000003, "nonpadding_tokens": 25335, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.43856886899993697, "nonpadding_tokens": 8151, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.43940686799999185, "nonpadding_tokens": 1466, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4394880480000438, "nonpadding_tokens": 8412, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4377739650000194, "nonpadding_tokens": 23473, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.43915107800000897, "nonpadding_tokens": 7161, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4384182569999666, "nonpadding_tokens": 2763, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.43845612299992354, "nonpadding_tokens": 2051, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003480540000282417, "prefetch": {"data_wait_seconds": 0.010963788000665886, "consumed_batches": 1168, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1718, "tokens": 132380955, "loss_tokens": 76282, "loss": 2.4164145866652684, "lr": 0.00029966088393582643, "optimizer_lrs": [0.01997739226238843, 0.00029966088393582643], "gradient_norm": 0.3857322037220001, "seconds": 0.2739669199999071, "nonpadding_tokens_per_second": 278435.07530042634, "ranks": [{"compute_seconds": 0.2564910559999589, "nonpadding_tokens": 8760, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2622003299999278, "nonpadding_tokens": 3649, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2626505890000317, "nonpadding_tokens": 3243, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2652676830000473, "nonpadding_tokens": 9131, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2617645499999526, "nonpadding_tokens": 18156, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2629349100000127, "nonpadding_tokens": 22455, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2616569099999424, "nonpadding_tokens": 4516, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26259732599999097, "nonpadding_tokens": 6372, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004029070000797219, "prefetch": {"data_wait_seconds": 0.010972813000606152, "consumed_batches": 1169, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1719, "tokens": 132438388, "loss_tokens": 57433, "loss": 2.5361608852924276, "lr": 0.0002996605373777975, "optimizer_lrs": [0.019977369158519834, 0.0002996605373777975], "gradient_norm": 0.45122087001800537, "seconds": 0.35500005000005785, "nonpadding_tokens_per_second": 161783.07580517422, "ranks": [{"compute_seconds": 0.35042060400007813, "nonpadding_tokens": 19376, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35250587399991673, "nonpadding_tokens": 13003, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3490974300000289, "nonpadding_tokens": 2061, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3495426539999471, "nonpadding_tokens": 7312, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35226230299997496, "nonpadding_tokens": 6035, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35124380700005986, "nonpadding_tokens": 2719, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3482442849999643, "nonpadding_tokens": 4380, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3489148190000151, "nonpadding_tokens": 2547, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005196140000407468, "prefetch": {"data_wait_seconds": 0.01098418500062337, "consumed_batches": 1170, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1720, "tokens": 132510554, "loss_tokens": 72166, "loss": 2.2560208408391764, "lr": 0.0002996601016683696, "optimizer_lrs": [0.019977340111224644, 0.0002996601016683696], "gradient_norm": 0.9812554717063904, "seconds": 0.26090246800004024, "nonpadding_tokens_per_second": 276601.44633046887, "ranks": [{"compute_seconds": 0.2585540159999482, "nonpadding_tokens": 14823, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2556983309999623, "nonpadding_tokens": 5735, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25622853199990914, "nonpadding_tokens": 3761, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2586589869999898, "nonpadding_tokens": 2358, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2557365299999219, "nonpadding_tokens": 10567, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2565902050000659, "nonpadding_tokens": 3486, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25568937000002734, "nonpadding_tokens": 21953, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25731930000006287, "nonpadding_tokens": 9483, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003417149999904723, "prefetch": {"data_wait_seconds": 0.010992513000658732, "consumed_batches": 1171, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1721, "tokens": 132563405, "loss_tokens": 52851, "loss": 2.7417190545117407, "lr": 0.0002996597823981417, "optimizer_lrs": [0.01997731882654278, 0.0002996597823981417], "gradient_norm": 0.3958728611469269, "seconds": 0.2592973119999442, "nonpadding_tokens_per_second": 203823.94091309118, "ranks": [{"compute_seconds": 0.25697754500004066, "nonpadding_tokens": 3449, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25673243100004584, "nonpadding_tokens": 3977, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25381204899997556, "nonpadding_tokens": 7930, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2560148370000661, "nonpadding_tokens": 3221, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2526677660000587, "nonpadding_tokens": 10342, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25530506200004766, "nonpadding_tokens": 5372, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.256638554999995, "nonpadding_tokens": 14683, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2531548910000083, "nonpadding_tokens": 3877, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039372799994907837, "prefetch": {"data_wait_seconds": 0.011002315000681051, "consumed_batches": 1172, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1722, "tokens": 132615313, "loss_tokens": 51908, "loss": 2.5462720702974493, "lr": 0.00029965946867882413, "optimizer_lrs": [0.01997729791192161, 0.00029965946867882413], "gradient_norm": 1.5157041549682617, "seconds": 0.24383709300002465, "nonpadding_tokens_per_second": 212879.83448849086, "ranks": [{"compute_seconds": 0.2402198940000062, "nonpadding_tokens": 2487, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24157580799999323, "nonpadding_tokens": 13935, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24002725900004407, "nonpadding_tokens": 2125, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23905024299995148, "nonpadding_tokens": 6037, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23663251300001775, "nonpadding_tokens": 6819, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23996923000004244, "nonpadding_tokens": 5909, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23754122399998323, "nonpadding_tokens": 10735, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24137663100009377, "nonpadding_tokens": 3861, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003614050000351199, "prefetch": {"data_wait_seconds": 0.011010815000602747, "consumed_batches": 1173, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1723, "tokens": 132692945, "loss_tokens": 77632, "loss": 2.895935511451463, "lr": 0.000299658999220453, "optimizer_lrs": [0.01997726661469687, 0.000299658999220453], "gradient_norm": 1.5773931741714478, "seconds": 0.2844323089999534, "nonpadding_tokens_per_second": 272936.64447948744, "ranks": [{"compute_seconds": 0.28023689999997714, "nonpadding_tokens": 4120, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28217672899995705, "nonpadding_tokens": 6955, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27909571199995753, "nonpadding_tokens": 19316, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27952464700001656, "nonpadding_tokens": 20601, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27870703600001434, "nonpadding_tokens": 8669, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.279482024999993, "nonpadding_tokens": 3931, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28171999599999253, "nonpadding_tokens": 3466, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2785310210000489, "nonpadding_tokens": 10574, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035567199995512055, "prefetch": {"data_wait_seconds": 0.011019522000609072, "consumed_batches": 1174, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1724, "tokens": 132756753, "loss_tokens": 63808, "loss": 2.615633766533977, "lr": 0.0002996586131171349, "optimizer_lrs": [0.019977240874475662, 0.0002996586131171349], "gradient_norm": 0.4398660659790039, "seconds": 0.3273461359999601, "nonpadding_tokens_per_second": 194925.16630777574, "ranks": [{"compute_seconds": 0.32477760999995553, "nonpadding_tokens": 11285, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32492062700009683, "nonpadding_tokens": 4162, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32150706300001275, "nonpadding_tokens": 2639, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3249373869999772, "nonpadding_tokens": 6909, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3209165619999794, "nonpadding_tokens": 11909, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3235835440000301, "nonpadding_tokens": 3789, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3206397899999729, "nonpadding_tokens": 3853, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32107043400003477, "nonpadding_tokens": 19262, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035425700002633675, "prefetch": {"data_wait_seconds": 0.011027868000681451, "consumed_batches": 1175, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1725, "tokens": 132836703, "loss_tokens": 79950, "loss": 2.440021497811132, "lr": 0.00029965812903033314, "optimizer_lrs": [0.01997720860202221, 0.00029965812903033314], "gradient_norm": 1.2654856443405151, "seconds": 0.2551204349999807, "nonpadding_tokens_per_second": 313381.40357124293, "ranks": [{"compute_seconds": 0.2508796559999382, "nonpadding_tokens": 5552, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25260119299991857, "nonpadding_tokens": 17852, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2514964360000249, "nonpadding_tokens": 10451, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2500182590000577, "nonpadding_tokens": 5431, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2525474360000999, "nonpadding_tokens": 4015, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2498075810000273, "nonpadding_tokens": 11187, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2521998930000109, "nonpadding_tokens": 6798, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2527855319999617, "nonpadding_tokens": 18664, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036600300006739417, "prefetch": {"data_wait_seconds": 0.011037223000656127, "consumed_batches": 1176, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1726, "tokens": 132925235, "loss_tokens": 88532, "loss": 2.755824518253287, "lr": 0.0002996575925809767, "optimizer_lrs": [0.01997717283873178, 0.0002996575925809767], "gradient_norm": 0.45354920625686646, "seconds": 0.3309281919999876, "nonpadding_tokens_per_second": 267526.31579966244, "ranks": [{"compute_seconds": 0.2960423309999669, "nonpadding_tokens": 19592, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30329222199998185, "nonpadding_tokens": 5564, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30626834299994243, "nonpadding_tokens": 18336, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30320542799995565, "nonpadding_tokens": 27457, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30102719000001343, "nonpadding_tokens": 7114, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30451788799996393, "nonpadding_tokens": 2540, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3010556460000089, "nonpadding_tokens": 4907, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3017969310000126, "nonpadding_tokens": 3022, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004242219999923691, "prefetch": {"data_wait_seconds": 0.0110470700006573, "consumed_batches": 1177, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1727, "tokens": 133000707, "loss_tokens": 75472, "loss": 2.508937089251643, "lr": 0.00029965713493551946, "optimizer_lrs": [0.019977142329034633, 0.00029965713493551946], "gradient_norm": 0.41579851508140564, "seconds": 0.3294877880000513, "nonpadding_tokens_per_second": 229058.56529040236, "ranks": [{"compute_seconds": 0.32340363600008004, "nonpadding_tokens": 10890, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32417974899999535, "nonpadding_tokens": 17088, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32585730599998897, "nonpadding_tokens": 13458, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32646886099996664, "nonpadding_tokens": 10092, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32711970799994106, "nonpadding_tokens": 13404, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3233794459999899, "nonpadding_tokens": 3280, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32006243499995435, "nonpadding_tokens": 5233, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3269085150000137, "nonpadding_tokens": 2027, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004157010000653827, "prefetch": {"data_wait_seconds": 0.01105687000070077, "consumed_batches": 1178, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1728, "tokens": 133063096, "loss_tokens": 62389, "loss": 2.678385562759461, "lr": 0.0002996567563920134, "optimizer_lrs": [0.019977117092800895, 0.0002996567563920134], "gradient_norm": 0.38088297843933105, "seconds": 0.28230672799998047, "nonpadding_tokens_per_second": 220997.21264880488, "ranks": [{"compute_seconds": 0.27655967399994097, "nonpadding_tokens": 2311, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2796978979999949, "nonpadding_tokens": 10002, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27988845000004403, "nonpadding_tokens": 12322, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2799000029999661, "nonpadding_tokens": 8335, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2758216510000011, "nonpadding_tokens": 7826, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2784326669999473, "nonpadding_tokens": 14521, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.279697730999942, "nonpadding_tokens": 2788, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27679523300002984, "nonpadding_tokens": 4284, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004296139999269144, "prefetch": {"data_wait_seconds": 0.01106633300067017, "consumed_batches": 1179, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1729, "tokens": 133113847, "loss_tokens": 50751, "loss": 2.5309759906208744, "lr": 0.0002996564483078948, "optimizer_lrs": [0.019977096553859656, 0.0002996564483078948], "gradient_norm": 0.5797318816184998, "seconds": 0.2101088570000229, "nonpadding_tokens_per_second": 241546.2190629806, "ranks": [{"compute_seconds": 0.20501098299996556, "nonpadding_tokens": 2714, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20755956100003914, "nonpadding_tokens": 11857, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20612788299990825, "nonpadding_tokens": 2704, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.20696848299996873, "nonpadding_tokens": 7298, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20751843099992584, "nonpadding_tokens": 2457, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20588705600005142, "nonpadding_tokens": 3726, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20704836600009457, "nonpadding_tokens": 16187, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20781713199994556, "nonpadding_tokens": 3808, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038804799999070383, "prefetch": {"data_wait_seconds": 0.01107558500071093, "consumed_batches": 1180, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1730, "tokens": 133164922, "loss_tokens": 51075, "loss": 2.7633351688693097, "lr": 0.0002996561381175836, "optimizer_lrs": [0.019977075874505573, 0.0002996561381175836], "gradient_norm": 0.41406238079071045, "seconds": 0.2633432249999714, "nonpadding_tokens_per_second": 193948.4108619295, "ranks": [{"compute_seconds": 0.2611205669999208, "nonpadding_tokens": 5609, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25777238599994234, "nonpadding_tokens": 5749, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2596247650000123, "nonpadding_tokens": 6645, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2602525830000104, "nonpadding_tokens": 7869, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25691831500000717, "nonpadding_tokens": 4892, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25834356099994693, "nonpadding_tokens": 8981, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2603026749999344, "nonpadding_tokens": 8641, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2573516350000773, "nonpadding_tokens": 2689, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003300849999732236, "prefetch": {"data_wait_seconds": 0.011083163000719765, "consumed_batches": 1181, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1731, "tokens": 133230264, "loss_tokens": 65342, "loss": 2.371807422867375, "lr": 0.0002996557410766555, "optimizer_lrs": [0.01997704940511037, 0.0002996557410766555], "gradient_norm": 0.6426030397415161, "seconds": 0.2614149250000537, "nonpadding_tokens_per_second": 249955.12402356742, "ranks": [{"compute_seconds": 0.2591024160000188, "nonpadding_tokens": 5605, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.259027385999957, "nonpadding_tokens": 15549, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.255957125000009, "nonpadding_tokens": 3501, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25840013899994574, "nonpadding_tokens": 2842, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2546910639999851, "nonpadding_tokens": 6863, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25777811000000384, "nonpadding_tokens": 9301, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2548417299999528, "nonpadding_tokens": 9933, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2553480940000554, "nonpadding_tokens": 11748, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038840599995637604, "prefetch": {"data_wait_seconds": 0.011091801000773671, "consumed_batches": 1182, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1732, "tokens": 133281757, "loss_tokens": 51493, "loss": 2.5466119181247935, "lr": 0.00029965542802591556, "optimizer_lrs": [0.019977028535061038, 0.00029965542802591556], "gradient_norm": 0.4647587835788727, "seconds": 0.23019291299999622, "nonpadding_tokens_per_second": 223694.9840414976, "ranks": [{"compute_seconds": 0.22686001600004602, "nonpadding_tokens": 1871, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22383272500007934, "nonpadding_tokens": 4230, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22425591299997905, "nonpadding_tokens": 14382, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22683002300004773, "nonpadding_tokens": 6615, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22788329700006216, "nonpadding_tokens": 10220, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22482515799993052, "nonpadding_tokens": 3751, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22324604599998565, "nonpadding_tokens": 2687, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22408557699998255, "nonpadding_tokens": 7737, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003664839999828473, "prefetch": {"data_wait_seconds": 0.011100560000684823, "consumed_batches": 1183, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1733, "tokens": 133323161, "loss_tokens": 41404, "loss": 2.384991108950826, "lr": 0.00029965517620800697, "optimizer_lrs": [0.019977011747200468, 0.00029965517620800697], "gradient_norm": 0.5034849047660828, "seconds": 0.27072631700002603, "nonpadding_tokens_per_second": 152936.7386917025, "ranks": [{"compute_seconds": 0.21677335300000777, "nonpadding_tokens": 2530, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22621710800001438, "nonpadding_tokens": 5323, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22635415600007036, "nonpadding_tokens": 3997, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22362304800003585, "nonpadding_tokens": 5546, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22739888000000974, "nonpadding_tokens": 12537, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22526872499997808, "nonpadding_tokens": 5040, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22675147400002515, "nonpadding_tokens": 4036, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22293677900006514, "nonpadding_tokens": 2395, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.002870934000043235, "prefetch": {"data_wait_seconds": 0.01111062400066487, "consumed_batches": 1184, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1734, "tokens": 133403423, "loss_tokens": 80262, "loss": 2.4636761870499115, "lr": 0.0002996546877952414, "optimizer_lrs": [0.019976979186349428, 0.0002996546877952414], "gradient_norm": 0.38452157378196716, "seconds": 0.2905726850000292, "nonpadding_tokens_per_second": 276220.0445647255, "ranks": [{"compute_seconds": 0.2876275359999454, "nonpadding_tokens": 10733, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2846256130000029, "nonpadding_tokens": 17782, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28488344799995957, "nonpadding_tokens": 3839, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28537575600000764, "nonpadding_tokens": 2962, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2876465609999741, "nonpadding_tokens": 11178, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28634941400002845, "nonpadding_tokens": 10409, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2837052700000413, "nonpadding_tokens": 14784, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2882181080000237, "nonpadding_tokens": 8575, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004084900000407288, "prefetch": {"data_wait_seconds": 0.011119898000586659, "consumed_batches": 1185, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1735, "tokens": 133461752, "loss_tokens": 58329, "loss": 2.440537457782578, "lr": 0.0002996543326332381, "optimizer_lrs": [0.01997695550888254, 0.0002996543326332381], "gradient_norm": 0.43958085775375366, "seconds": 0.2774469760000784, "nonpadding_tokens_per_second": 210234.76572324766, "ranks": [{"compute_seconds": 0.27505948199996055, "nonpadding_tokens": 7439, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2706972080000014, "nonpadding_tokens": 18828, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2719944430000396, "nonpadding_tokens": 5981, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27392069799998353, "nonpadding_tokens": 6227, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2745231069999363, "nonpadding_tokens": 8533, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2732615120000901, "nonpadding_tokens": 3192, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2743815489999406, "nonpadding_tokens": 5714, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27474599999993643, "nonpadding_tokens": 2415, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039869400006864453, "prefetch": {"data_wait_seconds": 0.011127882000508293, "consumed_batches": 1186, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1736, "tokens": 133494076, "loss_tokens": 32324, "loss": 2.655627397599307, "lr": 0.000299654135735706, "optimizer_lrs": [0.0199769423823804, 0.000299654135735706], "gradient_norm": 0.5995364189147949, "seconds": 0.22477635799998552, "nonpadding_tokens_per_second": 143805.15943764016, "ranks": [{"compute_seconds": 0.2217899459999444, "nonpadding_tokens": 5323, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2224931440000546, "nonpadding_tokens": 4674, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21932459100003143, "nonpadding_tokens": 1753, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22203555900000538, "nonpadding_tokens": 3715, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21821253500002058, "nonpadding_tokens": 5242, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2198941649999142, "nonpadding_tokens": 4574, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22212020300003132, "nonpadding_tokens": 3637, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21937690000004295, "nonpadding_tokens": 3406, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034838700003092526, "prefetch": {"data_wait_seconds": 0.011136168000533075, "consumed_batches": 1187, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1737, "tokens": 133568133, "loss_tokens": 74057, "loss": 2.848009177052811, "lr": 0.0002996536844158292, "optimizer_lrs": [0.019976912294388617, 0.0002996536844158292], "gradient_norm": 0.39628487825393677, "seconds": 0.3173762889999807, "nonpadding_tokens_per_second": 233341.31303049077, "ranks": [{"compute_seconds": 0.3145026570000482, "nonpadding_tokens": 18450, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3149996619999911, "nonpadding_tokens": 10945, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3122469889999593, "nonpadding_tokens": 13647, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3146481549999862, "nonpadding_tokens": 1033, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3150413910000225, "nonpadding_tokens": 4481, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31370047200005047, "nonpadding_tokens": 3836, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31089384999995673, "nonpadding_tokens": 10927, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3116648149999719, "nonpadding_tokens": 10738, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003295380000736259, "prefetch": {"data_wait_seconds": 0.011143807000465245, "consumed_batches": 1188, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1738, "tokens": 133624504, "loss_tokens": 56371, "loss": 2.541288406272729, "lr": 0.00029965334068138603, "optimizer_lrs": [0.01997688937875907, 0.00029965334068138603], "gradient_norm": 0.4170096814632416, "seconds": 0.2234441790000119, "nonpadding_tokens_per_second": 252282.24898173337, "ranks": [{"compute_seconds": 0.21989156899996942, "nonpadding_tokens": 5792, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2199953330000426, "nonpadding_tokens": 3239, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.219514815000025, "nonpadding_tokens": 4674, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2201626359999409, "nonpadding_tokens": 5757, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22120370200002526, "nonpadding_tokens": 9354, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21757101899993359, "nonpadding_tokens": 16340, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21595402699995248, "nonpadding_tokens": 4121, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21690061899994362, "nonpadding_tokens": 7094, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036593100003301515, "prefetch": {"data_wait_seconds": 0.011152522000500653, "consumed_batches": 1189, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1739, "tokens": 133676443, "loss_tokens": 51939, "loss": 2.4869168940969213, "lr": 0.0002996530238213008, "optimizer_lrs": [0.01997686825475339, 0.0002996530238213008], "gradient_norm": 0.4067123532295227, "seconds": 0.6449143490000324, "nonpadding_tokens_per_second": 80536.27598848385, "ranks": [{"compute_seconds": 0.6416357260000041, "nonpadding_tokens": 3786, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.6420121489999246, "nonpadding_tokens": 3963, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.6410466979999683, "nonpadding_tokens": 9451, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.6396808069999906, "nonpadding_tokens": 8147, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.642550976999928, "nonpadding_tokens": 10533, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.639327073000004, "nonpadding_tokens": 4767, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.637849668000058, "nonpadding_tokens": 9359, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.6385989890000019, "nonpadding_tokens": 1933, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003807649999316709, "prefetch": {"data_wait_seconds": 0.011161950000499132, "consumed_batches": 1190, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1740, "tokens": 133734156, "loss_tokens": 57713, "loss": 2.5654282830558106, "lr": 0.00029965267156668, "optimizer_lrs": [0.019976844771112, 0.00029965267156668], "gradient_norm": 1.5091710090637207, "seconds": 0.2582502770000019, "nonpadding_tokens_per_second": 223477.01102368836, "ranks": [{"compute_seconds": 0.25476947499998914, "nonpadding_tokens": 7329, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25521822999996857, "nonpadding_tokens": 5131, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2522565990000203, "nonpadding_tokens": 14067, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2549911550000843, "nonpadding_tokens": 13567, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2515998229999923, "nonpadding_tokens": 5680, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25409443600005943, "nonpadding_tokens": 4318, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25102405899997393, "nonpadding_tokens": 4247, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25592593199996827, "nonpadding_tokens": 3374, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000356536999902346, "prefetch": {"data_wait_seconds": 0.011171420000508192, "consumed_batches": 1191, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1741, "tokens": 133812312, "loss_tokens": 78156, "loss": 2.672918785825784, "lr": 0.0002996521942525135, "optimizer_lrs": [0.01997681295016757, 0.0002996521942525135], "gradient_norm": 0.41743233799934387, "seconds": 0.28213263099996766, "nonpadding_tokens_per_second": 277018.64801313594, "ranks": [{"compute_seconds": 0.2770821500000693, "nonpadding_tokens": 10300, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2759304840000141, "nonpadding_tokens": 6263, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27852571000005355, "nonpadding_tokens": 3312, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2787912139999662, "nonpadding_tokens": 4677, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27969582100001844, "nonpadding_tokens": 5201, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27800227399995947, "nonpadding_tokens": 21031, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2750010720000091, "nonpadding_tokens": 6512, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27559486300003755, "nonpadding_tokens": 20860, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004026129998919714, "prefetch": {"data_wait_seconds": 0.01118098700051462, "consumed_batches": 1192, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1742, "tokens": 133860673, "loss_tokens": 48361, "loss": 2.6412563196584022, "lr": 0.00029965189873837735, "optimizer_lrs": [0.019976793249225158, 0.00029965189873837735], "gradient_norm": 0.5129593014717102, "seconds": 0.21525554300001204, "nonpadding_tokens_per_second": 224667.84978446428, "ranks": [{"compute_seconds": 0.21286509900005512, "nonpadding_tokens": 4725, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21208333600009155, "nonpadding_tokens": 3014, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21155035800006772, "nonpadding_tokens": 5101, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21012370100004318, "nonpadding_tokens": 4797, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2124863029999915, "nonpadding_tokens": 6348, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21120958699998482, "nonpadding_tokens": 12274, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20838739900000292, "nonpadding_tokens": 2103, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.208870423999997, "nonpadding_tokens": 9999, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004763209999509854, "prefetch": {"data_wait_seconds": 0.011190230000536872, "consumed_batches": 1193, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1743, "tokens": 133995518, "loss_tokens": 134845, "loss": 2.28820706181171, "lr": 0.00029965107409425884, "optimizer_lrs": [0.01997673827295059, 0.00029965107409425884], "gradient_norm": 0.6673169136047363, "seconds": 0.28519320999998854, "nonpadding_tokens_per_second": 472819.81222486123, "ranks": [{"compute_seconds": 0.28260240299994166, "nonpadding_tokens": 14202, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28284625900005267, "nonpadding_tokens": 20612, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.279668235000031, "nonpadding_tokens": 5840, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27990880100003324, "nonpadding_tokens": 18247, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2827268380000305, "nonpadding_tokens": 19238, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28026146999991397, "nonpadding_tokens": 18335, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.282584509000003, "nonpadding_tokens": 19159, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2794884979999779, "nonpadding_tokens": 19212, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003495649999649686, "prefetch": {"data_wait_seconds": 0.011199224000577601, "consumed_batches": 1194, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1744, "tokens": 134059510, "loss_tokens": 63992, "loss": 2.253994884126141, "lr": 0.00029965068241049665, "optimizer_lrs": [0.019976712160699777, 0.00029965068241049665], "gradient_norm": 0.43821606040000916, "seconds": 0.36477904299999864, "nonpadding_tokens_per_second": 175426.74456767036, "ranks": [{"compute_seconds": 0.3614801459999626, "nonpadding_tokens": 11766, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3583185780000804, "nonpadding_tokens": 5909, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36091563100001167, "nonpadding_tokens": 4709, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3613793030000352, "nonpadding_tokens": 4790, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3576859989999548, "nonpadding_tokens": 3642, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36040205400001923, "nonpadding_tokens": 2083, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3579361900000322, "nonpadding_tokens": 12449, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36243621399989934, "nonpadding_tokens": 18644, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003603510000402821, "prefetch": {"data_wait_seconds": 0.011207344000652597, "consumed_batches": 1195, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1745, "tokens": 134111715, "loss_tokens": 52205, "loss": 2.4598948855473615, "lr": 0.0002996503627103581, "optimizer_lrs": [0.019976690847357207, 0.0002996503627103581], "gradient_norm": 0.3871951103210449, "seconds": 0.2776353479999898, "nonpadding_tokens_per_second": 188034.41411935026, "ranks": [{"compute_seconds": 0.27522485500003313, "nonpadding_tokens": 4518, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27530487199999243, "nonpadding_tokens": 9526, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27424723199999335, "nonpadding_tokens": 5256, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27291846699995403, "nonpadding_tokens": 2635, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27111570499994286, "nonpadding_tokens": 4549, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2725551880000694, "nonpadding_tokens": 3537, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27492210399998385, "nonpadding_tokens": 10698, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2718104159999939, "nonpadding_tokens": 11486, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003517000000101689, "prefetch": {"data_wait_seconds": 0.011215895000603382, "consumed_batches": 1196, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1746, "tokens": 134170592, "loss_tokens": 58877, "loss": 2.369682770861287, "lr": 0.00029965000197608124, "optimizer_lrs": [0.01997666679840542, 0.00029965000197608124], "gradient_norm": 0.4654187858104706, "seconds": 0.3318730089999917, "nonpadding_tokens_per_second": 177408.22062453855, "ranks": [{"compute_seconds": 0.3267939470000556, "nonpadding_tokens": 17275, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3287693309999895, "nonpadding_tokens": 6935, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3262097369999992, "nonpadding_tokens": 4626, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3285050769999316, "nonpadding_tokens": 5161, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3250272169999562, "nonpadding_tokens": 5278, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32642845899999884, "nonpadding_tokens": 13271, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3291891470000792, "nonpadding_tokens": 2738, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32952974500005894, "nonpadding_tokens": 3593, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035184499995466467, "prefetch": {"data_wait_seconds": 0.011223662000659829, "consumed_batches": 1197, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1747, "tokens": 134225646, "loss_tokens": 55054, "loss": 2.4467162354234024, "lr": 0.0002996496644969403, "optimizer_lrs": [0.019976644299796023, 0.0002996496644969403], "gradient_norm": 0.4247693717479706, "seconds": 0.28733295500001077, "nonpadding_tokens_per_second": 191603.50054520526, "ranks": [{"compute_seconds": 0.28506152199997814, "nonpadding_tokens": 11857, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28116243500005567, "nonpadding_tokens": 3046, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2818096080000032, "nonpadding_tokens": 2821, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.282596415999933, "nonpadding_tokens": 2438, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2806844289999617, "nonpadding_tokens": 2613, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28359151799998017, "nonpadding_tokens": 6556, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2846493910000163, "nonpadding_tokens": 18736, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28144905299996026, "nonpadding_tokens": 6987, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003426700000090932, "prefetch": {"data_wait_seconds": 0.011232954000618633, "consumed_batches": 1198, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1748, "tokens": 134304736, "loss_tokens": 79090, "loss": 2.093806699645973, "lr": 0.00029964917939368413, "optimizer_lrs": [0.019976611959578942, 0.00029964917939368413], "gradient_norm": 0.5225180387496948, "seconds": 0.30170197299992196, "nonpadding_tokens_per_second": 262146.11463618255, "ranks": [{"compute_seconds": 0.29915544799996496, "nonpadding_tokens": 4431, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2993712779999669, "nonpadding_tokens": 6291, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29608774000007543, "nonpadding_tokens": 2688, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29926693400000204, "nonpadding_tokens": 17907, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29534298600003694, "nonpadding_tokens": 18038, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2980615399999351, "nonpadding_tokens": 11785, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2954083439999522, "nonpadding_tokens": 6538, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2959515290000354, "nonpadding_tokens": 11412, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003989030000184357, "prefetch": {"data_wait_seconds": 0.011242298000595952, "consumed_batches": 1199, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1749, "tokens": 134410675, "loss_tokens": 105939, "loss": 2.5433990905143524, "lr": 0.0002996485290852944, "optimizer_lrs": [0.019976568605686296, 0.0002996485290852944], "gradient_norm": 0.4835672080516815, "seconds": 0.29277911000008316, "nonpadding_tokens_per_second": 361839.34024517634, "ranks": [{"compute_seconds": 0.2902180959999896, "nonpadding_tokens": 14101, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28993800299997474, "nonpadding_tokens": 25996, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28680868399999326, "nonpadding_tokens": 9919, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2895981039999924, "nonpadding_tokens": 7070, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28590286799999376, "nonpadding_tokens": 17495, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28889496900001177, "nonpadding_tokens": 18068, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28994829999999183, "nonpadding_tokens": 6017, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28684442600001603, "nonpadding_tokens": 7273, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003995690000238028, "prefetch": {"data_wait_seconds": 0.011250957000655148, "consumed_batches": 1200, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1750, "tokens": 134462602, "loss_tokens": 51927, "loss": 2.4531484704488995, "lr": 0.0002996482101108723, "optimizer_lrs": [0.01997654734072482, 0.0002996482101108723], "gradient_norm": 0.39183831214904785, "seconds": 0.27110606700000517, "nonpadding_tokens_per_second": 191537.58001291432, "ranks": [{"compute_seconds": 0.2674103669999113, "nonpadding_tokens": 8000, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2683526950000896, "nonpadding_tokens": 4492, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2648814139998876, "nonpadding_tokens": 3476, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.267781881000019, "nonpadding_tokens": 12170, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2682550709999987, "nonpadding_tokens": 4006, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2672229449999577, "nonpadding_tokens": 6935, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26438701500001116, "nonpadding_tokens": 9659, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26877486099999714, "nonpadding_tokens": 3189, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004135459998906299, "prefetch": {"data_wait_seconds": 0.011258905000730124, "consumed_batches": 1201, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1751, "tokens": 134531564, "loss_tokens": 68962, "loss": 2.431862837504713, "lr": 0.0002996477862713941, "optimizer_lrs": [0.019976519084759607, 0.0002996477862713941], "gradient_norm": 0.4318723976612091, "seconds": 0.27521060600008695, "nonpadding_tokens_per_second": 250579.00566513126, "ranks": [{"compute_seconds": 0.2719605940000065, "nonpadding_tokens": 3792, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2729953250000108, "nonpadding_tokens": 9142, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26965774300003886, "nonpadding_tokens": 4616, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2702925490000325, "nonpadding_tokens": 12488, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2689936019999095, "nonpadding_tokens": 9366, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27001776500003416, "nonpadding_tokens": 18588, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2691557489999923, "nonpadding_tokens": 5826, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2695787019999898, "nonpadding_tokens": 5144, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003457230000094569, "prefetch": {"data_wait_seconds": 0.011268292000707447, "consumed_batches": 1202, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1752, "tokens": 134612621, "loss_tokens": 81057, "loss": 2.3999943712449263, "lr": 0.0002996472877704197, "optimizer_lrs": [0.019976485851361313, 0.0002996472877704197], "gradient_norm": 0.39872556924819946, "seconds": 0.3467178829999966, "nonpadding_tokens_per_second": 233783.7301573533, "ranks": [{"compute_seconds": 0.3440796309999996, "nonpadding_tokens": 20070, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3442059159999644, "nonpadding_tokens": 10936, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34117720200003987, "nonpadding_tokens": 9808, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3418237969999609, "nonpadding_tokens": 3827, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3443169259999195, "nonpadding_tokens": 9494, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3432415159999209, "nonpadding_tokens": 17742, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3442743599999858, "nonpadding_tokens": 3481, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34082167499991556, "nonpadding_tokens": 5699, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003368630000295525, "prefetch": {"data_wait_seconds": 0.011277696000732362, "consumed_batches": 1203, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1753, "tokens": 134672887, "loss_tokens": 60266, "loss": 2.219324017688249, "lr": 0.00029964691690601545, "optimizer_lrs": [0.019976461127067698, 0.00029964691690601545], "gradient_norm": 0.4140946865081787, "seconds": 0.2858620790000259, "nonpadding_tokens_per_second": 210821.94676123705, "ranks": [{"compute_seconds": 0.2811414359999844, "nonpadding_tokens": 8040, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2832760620000272, "nonpadding_tokens": 4579, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27980770600004234, "nonpadding_tokens": 8262, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28238602499993704, "nonpadding_tokens": 18494, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2792404890000171, "nonpadding_tokens": 6144, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28155132099993807, "nonpadding_tokens": 4055, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28257459600001766, "nonpadding_tokens": 6733, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2835589970000001, "nonpadding_tokens": 3959, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038283900005353644, "prefetch": {"data_wait_seconds": 0.011288888000720476, "consumed_batches": 1204, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1754, "tokens": 134747182, "loss_tokens": 74295, "loss": 2.533093579648698, "lr": 0.00029964645944222237, "optimizer_lrs": [0.01997643062948149, 0.00029964645944222237], "gradient_norm": 0.39539191126823425, "seconds": 0.26554062999991856, "nonpadding_tokens_per_second": 279787.69200036465, "ranks": [{"compute_seconds": 0.2615019319999874, "nonpadding_tokens": 4717, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26284877800003414, "nonpadding_tokens": 6230, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2600919540000177, "nonpadding_tokens": 4027, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26081709400000364, "nonpadding_tokens": 4090, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26326922900000227, "nonpadding_tokens": 14397, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26144792399998096, "nonpadding_tokens": 10756, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2592360370000506, "nonpadding_tokens": 11718, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25987538999993376, "nonpadding_tokens": 18360, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033179999991261866, "prefetch": {"data_wait_seconds": 0.011297564000713578, "consumed_batches": 1205, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1755, "tokens": 134818993, "loss_tokens": 71811, "loss": 2.3291691906532423, "lr": 0.00029964601699230013, "optimizer_lrs": [0.01997640113282001, 0.00029964601699230013], "gradient_norm": 0.4581761360168457, "seconds": 0.35912294100000963, "nonpadding_tokens_per_second": 199962.1628182146, "ranks": [{"compute_seconds": 0.3550517820000323, "nonpadding_tokens": 17690, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35682345100008206, "nonpadding_tokens": 9115, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35359247100006996, "nonpadding_tokens": 2862, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.35399860199993327, "nonpadding_tokens": 2737, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35263533799991365, "nonpadding_tokens": 11069, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3539728629999672, "nonpadding_tokens": 7833, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35668346800002837, "nonpadding_tokens": 4836, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35371592700005294, "nonpadding_tokens": 15669, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003602450000244062, "prefetch": {"data_wait_seconds": 0.011306374000653818, "consumed_batches": 1206, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1756, "tokens": 134891991, "loss_tokens": 72998, "loss": 2.4727128654209705, "lr": 0.0002996455669456898, "optimizer_lrs": [0.019976371129712654, 0.0002996455669456898], "gradient_norm": 0.3595653176307678, "seconds": 0.3529549110000971, "nonpadding_tokens_per_second": 206819.61838457012, "ranks": [{"compute_seconds": 0.3496193049999192, "nonpadding_tokens": 10408, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35026496000000407, "nonpadding_tokens": 15241, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3494013230000519, "nonpadding_tokens": 5002, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3497220929999685, "nonpadding_tokens": 3986, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34644453700002487, "nonpadding_tokens": 13026, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3492187340000328, "nonpadding_tokens": 12164, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3501664880000135, "nonpadding_tokens": 5484, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35063827800001945, "nonpadding_tokens": 7687, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003578729999844654, "prefetch": {"data_wait_seconds": 0.011315173000596701, "consumed_batches": 1207, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1757, "tokens": 134958612, "loss_tokens": 66621, "loss": 2.5016544145239488, "lr": 0.00029964515596529615, "optimizer_lrs": [0.019976343731019745, 0.00029964515596529615], "gradient_norm": 1.691487193107605, "seconds": 0.3378975509999691, "nonpadding_tokens_per_second": 197163.31119548745, "ranks": [{"compute_seconds": 0.2674369910000678, "nonpadding_tokens": 11030, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27425299600008657, "nonpadding_tokens": 5085, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27484001699997407, "nonpadding_tokens": 7329, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27746133499999814, "nonpadding_tokens": 13789, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27778880100004244, "nonpadding_tokens": 7204, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2767178110000259, "nonpadding_tokens": 10203, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2776743350000288, "nonpadding_tokens": 10384, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27506636200007506, "nonpadding_tokens": 1597, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.038735137000003306, "prefetch": {"data_wait_seconds": 0.011323917000552797, "consumed_batches": 1208, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1758, "tokens": 135022273, "loss_tokens": 63661, "loss": 2.574481923783792, "lr": 0.0002996447630227321, "optimizer_lrs": [0.019976317534848808, 0.0002996447630227321], "gradient_norm": 0.39154455065727234, "seconds": 0.2867267820000734, "nonpadding_tokens_per_second": 222026.69578310862, "ranks": [{"compute_seconds": 0.2840474140000424, "nonpadding_tokens": 7597, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28029879500002153, "nonpadding_tokens": 9119, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.283203804999971, "nonpadding_tokens": 14127, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28200307400004476, "nonpadding_tokens": 3322, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2801717510000117, "nonpadding_tokens": 2320, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2812482210000553, "nonpadding_tokens": 19977, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2801747729999988, "nonpadding_tokens": 2756, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2843398669999715, "nonpadding_tokens": 4443, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040369600003486994, "prefetch": {"data_wait_seconds": 0.011331663000603953, "consumed_batches": 1209, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1759, "tokens": 135105625, "loss_tokens": 83352, "loss": 2.3338554038895287, "lr": 0.0002996442482107032, "optimizer_lrs": [0.019976283214046882, 0.0002996442482107032], "gradient_norm": 0.7534975409507751, "seconds": 0.35060682900007123, "nonpadding_tokens_per_second": 237736.38476386626, "ranks": [{"compute_seconds": 0.34831420800003343, "nonpadding_tokens": 15618, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34501698600001873, "nonpadding_tokens": 7462, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3475617259999808, "nonpadding_tokens": 8081, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34610193799994704, "nonpadding_tokens": 4583, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3445493650000344, "nonpadding_tokens": 12895, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3476278889999094, "nonpadding_tokens": 26075, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3483481559999291, "nonpadding_tokens": 4540, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3455806640000674, "nonpadding_tokens": 4098, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034718200004135724, "prefetch": {"data_wait_seconds": 0.011340385000565334, "consumed_batches": 1210, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1760, "tokens": 135172485, "loss_tokens": 66860, "loss": 2.9263245961711037, "lr": 0.0002996438349901444, "optimizer_lrs": [0.019976255666009628, 0.0002996438349901444], "gradient_norm": 0.4589304029941559, "seconds": 0.4367261280000321, "nonpadding_tokens_per_second": 153093.65690159737, "ranks": [{"compute_seconds": 0.43243087899998045, "nonpadding_tokens": 23381, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.42990708800004995, "nonpadding_tokens": 8287, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4291843590000326, "nonpadding_tokens": 10784, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4290469930000427, "nonpadding_tokens": 3452, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4273183699999663, "nonpadding_tokens": 3589, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.43016446199999336, "nonpadding_tokens": 3147, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.42809126000008746, "nonpadding_tokens": 4335, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4342859840000983, "nonpadding_tokens": 9885, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039388100003634463, "prefetch": {"data_wait_seconds": 0.011349575000579648, "consumed_batches": 1211, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1761, "tokens": 135240576, "loss_tokens": 68091, "loss": 2.4181596136053223, "lr": 0.00029964341391534535, "optimizer_lrs": [0.01997622759435636, 0.00029964341391534535], "gradient_norm": 0.6240419745445251, "seconds": 0.5120535070000187, "nonpadding_tokens_per_second": 132976.337568561, "ranks": [{"compute_seconds": 0.5091934879999371, "nonpadding_tokens": 14659, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5096183399999745, "nonpadding_tokens": 1642, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.5064479799999617, "nonpadding_tokens": 4341, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.5089710610000111, "nonpadding_tokens": 5489, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.5051385289999644, "nonpadding_tokens": 6317, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5081506679999848, "nonpadding_tokens": 26440, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.5093418279999469, "nonpadding_tokens": 3455, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.5094323469999154, "nonpadding_tokens": 5748, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004437090000237731, "prefetch": {"data_wait_seconds": 0.011359755000583505, "consumed_batches": 1212, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1762, "tokens": 135311204, "loss_tokens": 70628, "loss": 2.4497886372968227, "lr": 0.0002996429768892485, "optimizer_lrs": [0.019976198459283234, 0.0002996429768892485], "gradient_norm": 0.5346811413764954, "seconds": 0.2652841689999832, "nonpadding_tokens_per_second": 266235.2611022351, "ranks": [{"compute_seconds": 0.26276083000004746, "nonpadding_tokens": 11941, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25948477499991895, "nonpadding_tokens": 11255, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26001422199999524, "nonpadding_tokens": 16499, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2605123519999779, "nonpadding_tokens": 11752, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25898216000007324, "nonpadding_tokens": 3713, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2604001500000095, "nonpadding_tokens": 4542, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2589573799999698, "nonpadding_tokens": 5115, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25957740500007276, "nonpadding_tokens": 5811, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003992450000396275, "prefetch": {"data_wait_seconds": 0.011368712000560777, "consumed_batches": 1213, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1763, "tokens": 135378264, "loss_tokens": 67060, "loss": 2.7332570645690426, "lr": 0.00029964256169355276, "optimizer_lrs": [0.019976170779570184, 0.00029964256169355276], "gradient_norm": 0.5439210534095764, "seconds": 0.3089970870000798, "nonpadding_tokens_per_second": 217024.69965350413, "ranks": [{"compute_seconds": 0.30635009000002356, "nonpadding_tokens": 12087, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3032815799999753, "nonpadding_tokens": 6714, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30564613100000315, "nonpadding_tokens": 18528, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30453446599995004, "nonpadding_tokens": 8625, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30279819299994415, "nonpadding_tokens": 5797, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3036471900001061, "nonpadding_tokens": 4309, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3026385210000626, "nonpadding_tokens": 5674, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30332183399991663, "nonpadding_tokens": 5326, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041591399997287226, "prefetch": {"data_wait_seconds": 0.01137858000056724, "consumed_batches": 1214, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1764, "tokens": 135474271, "loss_tokens": 96007, "loss": 2.0290615202537317, "lr": 0.0002996419668557908, "optimizer_lrs": [0.01997613112371939, 0.0002996419668557908], "gradient_norm": 0.43278616666793823, "seconds": 0.3214460169999711, "nonpadding_tokens_per_second": 298672.2339757864, "ranks": [{"compute_seconds": 0.31902701000001343, "nonpadding_tokens": 15426, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3189501990000281, "nonpadding_tokens": 8962, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3161425570000347, "nonpadding_tokens": 18241, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31698974299990823, "nonpadding_tokens": 15755, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31486079799992694, "nonpadding_tokens": 3341, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3181071989999964, "nonpadding_tokens": 4832, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.319095602999937, "nonpadding_tokens": 16085, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3162120380000033, "nonpadding_tokens": 13365, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037990499993156845, "prefetch": {"data_wait_seconds": 0.011387832000608, "consumed_batches": 1215, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1765, "tokens": 135544473, "loss_tokens": 70202, "loss": 2.687143884789607, "lr": 0.00029964153158739564, "optimizer_lrs": [0.019976102105826378, 0.00029964153158739564], "gradient_norm": 0.37212833762168884, "seconds": 0.31487080399995193, "nonpadding_tokens_per_second": 222954.9361458445, "ranks": [{"compute_seconds": 0.307380460000104, "nonpadding_tokens": 14787, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3088336350000418, "nonpadding_tokens": 12844, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30940849099999923, "nonpadding_tokens": 11942, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31007491999992, "nonpadding_tokens": 3065, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31244367700003295, "nonpadding_tokens": 4087, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3111057630000005, "nonpadding_tokens": 12715, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3083822450000753, "nonpadding_tokens": 4554, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31258505799996783, "nonpadding_tokens": 6208, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038291600003503845, "prefetch": {"data_wait_seconds": 0.011396880000646092, "consumed_batches": 1216, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1766, "tokens": 135590583, "loss_tokens": 46110, "loss": 2.4221044106484495, "lr": 0.00029964124555120514, "optimizer_lrs": [0.01997608303674701, 0.00029964124555120514], "gradient_norm": 0.4205934703350067, "seconds": 0.3589222580000069, "nonpadding_tokens_per_second": 128467.93134796091, "ranks": [{"compute_seconds": 0.35520891500004836, "nonpadding_tokens": 4169, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35284446799994384, "nonpadding_tokens": 6510, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35284247399999913, "nonpadding_tokens": 13877, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3556042399999342, "nonpadding_tokens": 3685, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35651558699998986, "nonpadding_tokens": 3572, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3534235309999758, "nonpadding_tokens": 3860, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35168407599996954, "nonpadding_tokens": 6681, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3527216559999715, "nonpadding_tokens": 3756, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034128999993754405, "prefetch": {"data_wait_seconds": 0.011405327000673537, "consumed_batches": 1217, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1767, "tokens": 135634400, "loss_tokens": 43817, "loss": 2.696145510018942, "lr": 0.000299640973633717, "optimizer_lrs": [0.019976064908914466, 0.000299640973633717], "gradient_norm": 0.3918246924877167, "seconds": 0.17175025599999572, "nonpadding_tokens_per_second": 255120.43487143968, "ranks": [{"compute_seconds": 0.16919014500001595, "nonpadding_tokens": 5136, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.16915209499995854, "nonpadding_tokens": 6642, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.16574074400000427, "nonpadding_tokens": 10205, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.16830234099995778, "nonpadding_tokens": 5108, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1688500779999913, "nonpadding_tokens": 3942, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.16600512699994852, "nonpadding_tokens": 3641, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.16464694300009342, "nonpadding_tokens": 3899, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.16538251399992987, "nonpadding_tokens": 5244, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044948400000066613, "prefetch": {"data_wait_seconds": 0.011415551000666824, "consumed_batches": 1218, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1768, "tokens": 135687313, "loss_tokens": 52913, "loss": 2.4139203149509574, "lr": 0.00029964064513157414, "optimizer_lrs": [0.01997604300877161, 0.00029964064513157414], "gradient_norm": 0.4146146774291992, "seconds": 0.2944396829999505, "nonpadding_tokens_per_second": 179707.4343406656, "ranks": [{"compute_seconds": 0.2879727439999442, "nonpadding_tokens": 8376, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29062936799994077, "nonpadding_tokens": 5991, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29052525200006585, "nonpadding_tokens": 9222, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2880345669999542, "nonpadding_tokens": 3551, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2917602090000173, "nonpadding_tokens": 3941, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2890489650000063, "nonpadding_tokens": 7252, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2912842800000135, "nonpadding_tokens": 4867, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29213297700005114, "nonpadding_tokens": 9713, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038397399998757464, "prefetch": {"data_wait_seconds": 0.011435453000743223, "consumed_batches": 1219, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1769, "tokens": 135745636, "loss_tokens": 58323, "loss": 2.6798861512610803, "lr": 0.00029964028286847745, "optimizer_lrs": [0.019976018857898497, 0.00029964028286847745], "gradient_norm": 0.41521263122558594, "seconds": 0.3073555690000376, "nonpadding_tokens_per_second": 189757.420663469, "ranks": [{"compute_seconds": 0.3048259799999187, "nonpadding_tokens": 7753, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3014928359999658, "nonpadding_tokens": 6537, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3017697060000728, "nonpadding_tokens": 4777, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30279852599994683, "nonpadding_tokens": 2784, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30503694499998346, "nonpadding_tokens": 3544, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30218749599998773, "nonpadding_tokens": 11946, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3009720649999963, "nonpadding_tokens": 12530, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30148254200003066, "nonpadding_tokens": 8452, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039367800002310105, "prefetch": {"data_wait_seconds": 0.011443779000799736, "consumed_batches": 1220, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1770, "tokens": 135800365, "loss_tokens": 54729, "loss": 2.32574323599006, "lr": 0.0002996399427631213, "optimizer_lrs": [0.01997599618420809, 0.0002996399427631213], "gradient_norm": 0.4248580038547516, "seconds": 0.27634658199997375, "nonpadding_tokens_per_second": 198044.78710724635, "ranks": [{"compute_seconds": 0.2738906480000196, "nonpadding_tokens": 3866, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2700737710000567, "nonpadding_tokens": 5392, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2703835519999984, "nonpadding_tokens": 6457, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27323250899996765, "nonpadding_tokens": 14623, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27293204600005083, "nonpadding_tokens": 5481, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2701226909999832, "nonpadding_tokens": 4218, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26851438500000313, "nonpadding_tokens": 4071, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2732245710000143, "nonpadding_tokens": 10621, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003983849999258382, "prefetch": {"data_wait_seconds": 0.011452300000726723, "consumed_batches": 1221, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1771, "tokens": 135884959, "loss_tokens": 84594, "loss": 2.2895933443861267, "lr": 0.0002996394167503432, "optimizer_lrs": [0.019975961116689547, 0.0002996394167503432], "gradient_norm": 0.5806638598442078, "seconds": 0.25772123299998384, "nonpadding_tokens_per_second": 328238.37995531125, "ranks": [{"compute_seconds": 0.25527177399999346, "nonpadding_tokens": 11765, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2518308019999722, "nonpadding_tokens": 17067, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25442028899999514, "nonpadding_tokens": 11074, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25303040000005694, "nonpadding_tokens": 23964, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2518929280000748, "nonpadding_tokens": 10567, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2542320259999542, "nonpadding_tokens": 3402, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2554279390000147, "nonpadding_tokens": 1738, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25210273100003633, "nonpadding_tokens": 5017, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038489099995331344, "prefetch": {"data_wait_seconds": 0.011462184000677667, "consumed_batches": 1222, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1772, "tokens": 135956209, "loss_tokens": 71250, "loss": 2.891002850877193, "lr": 0.00029963897341422603, "optimizer_lrs": [0.019975931560948405, 0.00029963897341422603], "gradient_norm": 0.4602757692337036, "seconds": 0.3361850910000612, "nonpadding_tokens_per_second": 211936.82262366306, "ranks": [{"compute_seconds": 0.3338861359999328, "nonpadding_tokens": 5740, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33341832700000396, "nonpadding_tokens": 11755, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3326788069999793, "nonpadding_tokens": 2913, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.331231864000074, "nonpadding_tokens": 13573, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3334433740000122, "nonpadding_tokens": 10001, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3321892180000532, "nonpadding_tokens": 14114, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3332861349999803, "nonpadding_tokens": 3015, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3298045300000467, "nonpadding_tokens": 10139, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003601480000270385, "prefetch": {"data_wait_seconds": 0.011471400000644394, "consumed_batches": 1223, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1773, "tokens": 136033750, "loss_tokens": 77541, "loss": 2.4881766178537807, "lr": 0.0002996384906247696, "optimizer_lrs": [0.019975899374984642, 0.0002996384906247696], "gradient_norm": 0.41239243745803833, "seconds": 0.26845610600003056, "nonpadding_tokens_per_second": 288840.5153280111, "ranks": [{"compute_seconds": 0.2637981599999648, "nonpadding_tokens": 7600, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2639069610000888, "nonpadding_tokens": 20015, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26100298199992267, "nonpadding_tokens": 7853, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2612311300000556, "nonpadding_tokens": 3797, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2660697200000186, "nonpadding_tokens": 6473, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26347977399996125, "nonpadding_tokens": 8821, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2657051250000677, "nonpadding_tokens": 16736, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26088980899999115, "nonpadding_tokens": 6246, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003916260000096372, "prefetch": {"data_wait_seconds": 0.011481194000637629, "consumed_batches": 1224, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1774, "tokens": 136088404, "loss_tokens": 54654, "loss": 2.3186010230266767, "lr": 0.0002996381501418786, "optimizer_lrs": [0.019975876676125243, 0.0002996381501418786], "gradient_norm": 0.40406063199043274, "seconds": 0.28550976300005004, "nonpadding_tokens_per_second": 191426.02839816172, "ranks": [{"compute_seconds": 0.2808459469999889, "nonpadding_tokens": 9924, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.279175652000049, "nonpadding_tokens": 5032, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27877181200005907, "nonpadding_tokens": 4924, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2793209050000769, "nonpadding_tokens": 6088, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2779121909999276, "nonpadding_tokens": 9758, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2810813749999852, "nonpadding_tokens": 8062, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2823156929999868, "nonpadding_tokens": 5257, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28310571500003334, "nonpadding_tokens": 5609, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003811889999951745, "prefetch": {"data_wait_seconds": 0.011489989000622813, "consumed_batches": 1225, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1775, "tokens": 136126756, "loss_tokens": 38352, "loss": 2.646164150500626, "lr": 0.0002996379111214483, "optimizer_lrs": [0.019975860741429888, 0.0002996379111214483], "gradient_norm": 0.4553944766521454, "seconds": 0.2637389340000027, "nonpadding_tokens_per_second": 145416.52769400974, "ranks": [{"compute_seconds": 0.26130945099998826, "nonpadding_tokens": 5961, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2592122379999182, "nonpadding_tokens": 6463, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26143906300001163, "nonpadding_tokens": 1809, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2595976300000302, "nonpadding_tokens": 8611, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.257829313000002, "nonpadding_tokens": 4789, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2597165549999545, "nonpadding_tokens": 2062, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2578621950000297, "nonpadding_tokens": 5799, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2587369289999515, "nonpadding_tokens": 2858, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033984399999553716, "prefetch": {"data_wait_seconds": 0.011498950000557784, "consumed_batches": 1226, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1776, "tokens": 136207473, "loss_tokens": 80717, "loss": 2.3029721124422364, "lr": 0.0002996374078129661, "optimizer_lrs": [0.019975827187531076, 0.0002996374078129661], "gradient_norm": 0.6520235538482666, "seconds": 0.2902782890000708, "nonpadding_tokens_per_second": 278067.644252858, "ranks": [{"compute_seconds": 0.28544266899996273, "nonpadding_tokens": 6735, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28299068100000113, "nonpadding_tokens": 10217, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28538135099995543, "nonpadding_tokens": 5796, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28632884799992553, "nonpadding_tokens": 6379, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2807164289999946, "nonpadding_tokens": 10559, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28389593700001114, "nonpadding_tokens": 20357, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.280689553000002, "nonpadding_tokens": 9134, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28791453799999545, "nonpadding_tokens": 11540, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003494490000548467, "prefetch": {"data_wait_seconds": 0.01150776200059056, "consumed_batches": 1227, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1777, "tokens": 136273972, "loss_tokens": 66499, "loss": 2.5790479462097173, "lr": 0.00029963699289813626, "optimizer_lrs": [0.01997579952654242, 0.00029963699289813626], "gradient_norm": 0.4345739483833313, "seconds": 0.4214169789999005, "nonpadding_tokens_per_second": 157798.57792586877, "ranks": [{"compute_seconds": 0.4182378709999739, "nonpadding_tokens": 12297, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.41831306899996434, "nonpadding_tokens": 3830, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.415057016999981, "nonpadding_tokens": 3346, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4158742649999567, "nonpadding_tokens": 6832, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4147077899999658, "nonpadding_tokens": 8740, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4170545870000524, "nonpadding_tokens": 10372, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.41457044300000234, "nonpadding_tokens": 13595, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.419011490999992, "nonpadding_tokens": 7487, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003992210000660634, "prefetch": {"data_wait_seconds": 0.011515900000588886, "consumed_batches": 1228, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1778, "tokens": 136341823, "loss_tokens": 67851, "loss": 2.4941830260423576, "lr": 0.00029963656930340443, "optimizer_lrs": [0.01997577128689363, 0.00029963656930340443], "gradient_norm": 0.41066640615463257, "seconds": 0.2771726180000087, "nonpadding_tokens_per_second": 244796.9084738301, "ranks": [{"compute_seconds": 0.27481173800003944, "nonpadding_tokens": 7550, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2741294040000639, "nonpadding_tokens": 7407, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2711518920000344, "nonpadding_tokens": 3229, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2717575849999321, "nonpadding_tokens": 6620, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27405829999997877, "nonpadding_tokens": 18576, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2728405049999765, "nonpadding_tokens": 7827, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27037059100007355, "nonpadding_tokens": 12028, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2714549579999357, "nonpadding_tokens": 4614, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003386339999451593, "prefetch": {"data_wait_seconds": 0.011523772000671784, "consumed_batches": 1229, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1779, "tokens": 136415424, "loss_tokens": 73601, "loss": 2.5321893979022025, "lr": 0.0002996361095324313, "optimizer_lrs": [0.019975740635495422, 0.0002996361095324313], "gradient_norm": 0.4024316370487213, "seconds": 0.2927808700000014, "nonpadding_tokens_per_second": 251385.95974525125, "ranks": [{"compute_seconds": 0.2687836509999215, "nonpadding_tokens": 6919, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2764383610000323, "nonpadding_tokens": 7633, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2769728110000642, "nonpadding_tokens": 4370, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27964763400007087, "nonpadding_tokens": 6377, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2758630199999743, "nonpadding_tokens": 9702, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27868638200004625, "nonpadding_tokens": 1757, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27607078599999113, "nonpadding_tokens": 19309, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28030597400004353, "nonpadding_tokens": 17534, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.006198601999926723, "prefetch": {"data_wait_seconds": 0.011531781000670094, "consumed_batches": 1230, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1780, "tokens": 136453441, "loss_tokens": 38017, "loss": 2.250748431622695, "lr": 0.00029963587193402826, "optimizer_lrs": [0.019975724795601887, 0.00029963587193402826], "gradient_norm": 0.4800769090652466, "seconds": 0.32009601299989754, "nonpadding_tokens_per_second": 118767.48992813031, "ranks": [{"compute_seconds": 0.31619103499997436, "nonpadding_tokens": 13689, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31399098400004277, "nonpadding_tokens": 1318, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3149593970000524, "nonpadding_tokens": 1896, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3170764149999741, "nonpadding_tokens": 8316, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3177507579999883, "nonpadding_tokens": 2272, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.314726276999977, "nonpadding_tokens": 3614, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31412317499996334, "nonpadding_tokens": 2626, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.314370112000006, "nonpadding_tokens": 4286, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037162500007070776, "prefetch": {"data_wait_seconds": 0.011541289000732036, "consumed_batches": 1231, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1781, "tokens": 136579050, "loss_tokens": 125609, "loss": 2.4825868369304747, "lr": 0.00029963508635309446, "optimizer_lrs": [0.019975672423539634, 0.00029963508635309446], "gradient_norm": 0.7778489589691162, "seconds": 0.3329207230000293, "nonpadding_tokens_per_second": 377294.0262417637, "ranks": [{"compute_seconds": 0.3304970780000076, "nonpadding_tokens": 26108, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3300015570000596, "nonpadding_tokens": 19730, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32685067199997775, "nonpadding_tokens": 27014, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3294422599999507, "nonpadding_tokens": 10072, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.329873120000002, "nonpadding_tokens": 15387, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32745406499998353, "nonpadding_tokens": 18317, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3301303179999877, "nonpadding_tokens": 6045, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3264844060000769, "nonpadding_tokens": 2936, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003904930000544482, "prefetch": {"data_wait_seconds": 0.011551718000717415, "consumed_batches": 1232, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1782, "tokens": 136625625, "loss_tokens": 46575, "loss": 2.5786980005367686, "lr": 0.0002996347948499549, "optimizer_lrs": [0.019975652989996994, 0.0002996347948499549], "gradient_norm": 0.47684982419013977, "seconds": 0.2083683440000641, "nonpadding_tokens_per_second": 223522.4367862024, "ranks": [{"compute_seconds": 0.17136913499996353, "nonpadding_tokens": 5176, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.18458598899997014, "nonpadding_tokens": 8170, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.18338883499995973, "nonpadding_tokens": 6822, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.1839918209999496, "nonpadding_tokens": 3368, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.18435564399999294, "nonpadding_tokens": 2916, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1832975139999462, "nonpadding_tokens": 3255, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.18099241100003383, "nonpadding_tokens": 11245, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.18496390600000723, "nonpadding_tokens": 5623, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.006896138999991308, "prefetch": {"data_wait_seconds": 0.011561555000710655, "consumed_batches": 1233, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1783, "tokens": 136691434, "loss_tokens": 65809, "loss": 2.45418084912398, "lr": 0.0002996343827671169, "optimizer_lrs": [0.019975625517807793, 0.0002996343827671169], "gradient_norm": 0.7571374177932739, "seconds": 0.31780884699992384, "nonpadding_tokens_per_second": 207071.01335041114, "ranks": [{"compute_seconds": 0.31334570199999234, "nonpadding_tokens": 6697, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31161352399999487, "nonpadding_tokens": 5065, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3122244090000095, "nonpadding_tokens": 2811, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31311970799993105, "nonpadding_tokens": 10771, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3125673970000662, "nonpadding_tokens": 10408, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3124496120000231, "nonpadding_tokens": 7567, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31505655000000843, "nonpadding_tokens": 6552, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31554538700004287, "nonpadding_tokens": 15938, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036931399995410175, "prefetch": {"data_wait_seconds": 0.011569509000722178, "consumed_batches": 1234, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1784, "tokens": 136762198, "loss_tokens": 70764, "loss": 2.6406545877847494, "lr": 0.0002996339393981495, "optimizer_lrs": [0.019975595959876635, 0.0002996339393981495], "gradient_norm": 0.39524543285369873, "seconds": 0.27386306400001104, "nonpadding_tokens_per_second": 258391.90932296423, "ranks": [{"compute_seconds": 0.27127253899993775, "nonpadding_tokens": 2057, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2712195789999896, "nonpadding_tokens": 6340, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.267640756999981, "nonpadding_tokens": 9792, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26871320999998716, "nonpadding_tokens": 12236, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26699846299993624, "nonpadding_tokens": 9985, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26990370600003644, "nonpadding_tokens": 15652, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26690292199998567, "nonpadding_tokens": 7587, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27059186999997564, "nonpadding_tokens": 7115, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038615200003278005, "prefetch": {"data_wait_seconds": 0.011578490000786701, "consumed_batches": 1235, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1785, "tokens": 136815145, "loss_tokens": 52947, "loss": 2.5131694194194196, "lr": 0.0002996336074853968, "optimizer_lrs": [0.019975573832359785, 0.0002996336074853968], "gradient_norm": 0.4239577651023865, "seconds": 0.313063975000091, "nonpadding_tokens_per_second": 169125.17641157727, "ranks": [{"compute_seconds": 0.3101249359999656, "nonpadding_tokens": 14258, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30658064500005366, "nonpadding_tokens": 3120, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30719235399999434, "nonpadding_tokens": 12107, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3098833740000373, "nonpadding_tokens": 4694, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30630474200006574, "nonpadding_tokens": 4074, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30907843800002865, "nonpadding_tokens": 4265, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31028754800001934, "nonpadding_tokens": 3661, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3105938960000003, "nonpadding_tokens": 6768, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004712789999530287, "prefetch": {"data_wait_seconds": 0.01158931400080121, "consumed_batches": 1236, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1786, "tokens": 136854618, "loss_tokens": 39473, "loss": 2.4006734033389914, "lr": 0.0002996333599403681, "optimizer_lrs": [0.019975557329357872, 0.0002996333599403681], "gradient_norm": 0.41734200716018677, "seconds": 0.19021474199996646, "nonpadding_tokens_per_second": 207518.09026456508, "ranks": [{"compute_seconds": 0.1866437840000117, "nonpadding_tokens": 1329, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.18763271300008455, "nonpadding_tokens": 3185, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.18396723799992287, "nonpadding_tokens": 4017, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.18509962600001018, "nonpadding_tokens": 2016, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.18312952400003724, "nonpadding_tokens": 7418, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1851200930000232, "nonpadding_tokens": 8067, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.1830474170000116, "nonpadding_tokens": 5887, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.18792369900006634, "nonpadding_tokens": 7554, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035755500005052454, "prefetch": {"data_wait_seconds": 0.011598358000767917, "consumed_batches": 1237, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1787, "tokens": 136928033, "loss_tokens": 73415, "loss": 2.150643175781516, "lr": 0.00029963289931456935, "optimizer_lrs": [0.019975526620971292, 0.00029963289931456935], "gradient_norm": 0.6368510127067566, "seconds": 0.3305077730000221, "nonpadding_tokens_per_second": 222127.90741231703, "ranks": [{"compute_seconds": 0.32749233300000924, "nonpadding_tokens": 25097, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3282135220000555, "nonpadding_tokens": 6850, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3251482460000261, "nonpadding_tokens": 2478, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3279457739999998, "nonpadding_tokens": 3506, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3243052309999257, "nonpadding_tokens": 7532, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3254871709999634, "nonpadding_tokens": 9918, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3280604169999606, "nonpadding_tokens": 2813, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3256370330000209, "nonpadding_tokens": 15221, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003927259999727539, "prefetch": {"data_wait_seconds": 0.01160568000075557, "consumed_batches": 1238, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1788, "tokens": 136999205, "loss_tokens": 71172, "loss": 2.5241711979430113, "lr": 0.0002996324524862892, "optimizer_lrs": [0.01997549683241928, 0.0002996324524862892], "gradient_norm": 0.40724098682403564, "seconds": 0.2326379869999755, "nonpadding_tokens_per_second": 305934.55917415366, "ranks": [{"compute_seconds": 0.22900995300005889, "nonpadding_tokens": 8649, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23039986599997064, "nonpadding_tokens": 5350, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2295421600000509, "nonpadding_tokens": 10032, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22971830400001636, "nonpadding_tokens": 8391, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22643890699998792, "nonpadding_tokens": 7810, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2292533419999927, "nonpadding_tokens": 13379, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22632738300001165, "nonpadding_tokens": 14408, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2268655889999991, "nonpadding_tokens": 3153, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035636400002658775, "prefetch": {"data_wait_seconds": 0.01161470600068526, "consumed_batches": 1239, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1789, "tokens": 137060956, "loss_tokens": 61751, "loss": 2.311396019902512, "lr": 0.0002996320645845424, "optimizer_lrs": [0.019975470972302827, 0.0002996320645845424], "gradient_norm": 0.399214506149292, "seconds": 0.202036061000058, "nonpadding_tokens_per_second": 305643.4563925807, "ranks": [{"compute_seconds": 0.197654349000004, "nonpadding_tokens": 3719, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19633543999998437, "nonpadding_tokens": 4259, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.19879848499999753, "nonpadding_tokens": 5538, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.19771140199998172, "nonpadding_tokens": 11399, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.19575962199996866, "nonpadding_tokens": 18639, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19699917700006608, "nonpadding_tokens": 2557, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19972534900000483, "nonpadding_tokens": 4660, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.1961027520000016, "nonpadding_tokens": 10980, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038554700006443454, "prefetch": {"data_wait_seconds": 0.011623668000609655, "consumed_batches": 1240, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1790, "tokens": 137097856, "loss_tokens": 36900, "loss": 2.451914591802168, "lr": 0.00029963183269200093, "optimizer_lrs": [0.019975455512800062, 0.00029963183269200093], "gradient_norm": 0.41250184178352356, "seconds": 0.2157601669999849, "nonpadding_tokens_per_second": 171023.22691473714, "ranks": [{"compute_seconds": 0.21320627299996886, "nonpadding_tokens": 2582, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21232547099998555, "nonpadding_tokens": 2437, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2109429119999504, "nonpadding_tokens": 4034, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21197912699994959, "nonpadding_tokens": 5713, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21269104099997094, "nonpadding_tokens": 4721, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2089101200000414, "nonpadding_tokens": 10036, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21219731199994385, "nonpadding_tokens": 4937, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2083625430000211, "nonpadding_tokens": 2440, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039852899999459623, "prefetch": {"data_wait_seconds": 0.011632625000586927, "consumed_batches": 1241, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1791, "tokens": 137156605, "loss_tokens": 58749, "loss": 2.786213914279392, "lr": 0.0002996314633421449, "optimizer_lrs": [0.01997543088947633, 0.0002996314633421449], "gradient_norm": 0.3974718749523163, "seconds": 0.320060359000081, "nonpadding_tokens_per_second": 183556.00232262793, "ranks": [{"compute_seconds": 0.31763557699991907, "nonpadding_tokens": 12653, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3171741870000915, "nonpadding_tokens": 5688, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31672282099998483, "nonpadding_tokens": 1699, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.315009459999942, "nonpadding_tokens": 13710, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31757199899993793, "nonpadding_tokens": 10217, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31484278799996446, "nonpadding_tokens": 7103, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3171783249999862, "nonpadding_tokens": 4608, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31774457300002723, "nonpadding_tokens": 3071, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003521309999996447, "prefetch": {"data_wait_seconds": 0.011640856000511235, "consumed_batches": 1242, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1792, "tokens": 137199943, "loss_tokens": 43338, "loss": 2.342264042237759, "lr": 0.00029963119076138473, "optimizer_lrs": [0.01997541271742565, 0.00029963119076138473], "gradient_norm": 0.6062316298484802, "seconds": 0.21046838999995998, "nonpadding_tokens_per_second": 205912.15621504132, "ranks": [{"compute_seconds": 0.19105217100002392, "nonpadding_tokens": 5426, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19249278699999195, "nonpadding_tokens": 2263, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.19105164300003707, "nonpadding_tokens": 2515, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.18997553200006223, "nonpadding_tokens": 11588, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1924062049999975, "nonpadding_tokens": 6454, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.18956371399997352, "nonpadding_tokens": 7282, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19200317199999972, "nonpadding_tokens": 3864, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.1892836439999428, "nonpadding_tokens": 3946, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0061680249999653824, "prefetch": {"data_wait_seconds": 0.011649878000596914, "consumed_batches": 1243, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1793, "tokens": 137235514, "loss_tokens": 35571, "loss": 2.2997082419667705, "lr": 0.00029963096695712635, "optimizer_lrs": [0.019975397797141758, 0.00029963096695712635], "gradient_norm": 0.44304007291793823, "seconds": 0.22476023499996245, "nonpadding_tokens_per_second": 158261.9808170513, "ranks": [{"compute_seconds": 0.16799750999996377, "nonpadding_tokens": 2102, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.17434037999998964, "nonpadding_tokens": 3332, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.1751596870000185, "nonpadding_tokens": 4018, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.17764299199996003, "nonpadding_tokens": 3109, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.17421139800001129, "nonpadding_tokens": 1997, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1767405380001037, "nonpadding_tokens": 4321, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.17370538800003033, "nonpadding_tokens": 11247, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.17826496800000768, "nonpadding_tokens": 5445, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.03450539400000707, "prefetch": {"data_wait_seconds": 0.011657358000618956, "consumed_batches": 1244, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1794, "tokens": 137281639, "loss_tokens": 46125, "loss": 2.722484756097561, "lr": 0.0002996306766486672, "optimizer_lrs": [0.01997537844324448, 0.0002996306766486672], "gradient_norm": 0.44580209255218506, "seconds": 0.22493232900001203, "nonpadding_tokens_per_second": 205061.67434916628, "ranks": [{"compute_seconds": 0.22196860700000798, "nonpadding_tokens": 4242, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22133669900006225, "nonpadding_tokens": 9119, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22069810000004964, "nonpadding_tokens": 4782, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21899739599996337, "nonpadding_tokens": 12299, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22207106600001225, "nonpadding_tokens": 2447, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2190020259999983, "nonpadding_tokens": 2548, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22033747000000403, "nonpadding_tokens": 5454, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22261340599993673, "nonpadding_tokens": 5234, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038865900000928377, "prefetch": {"data_wait_seconds": 0.011665841000606747, "consumed_batches": 1245, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1795, "tokens": 137351520, "loss_tokens": 69881, "loss": 2.255537762410383, "lr": 0.0002996302366039566, "optimizer_lrs": [0.01997534910693044, 0.0002996302366039566], "gradient_norm": 0.5581671595573425, "seconds": 0.2758537019999494, "nonpadding_tokens_per_second": 253326.30845031334, "ranks": [{"compute_seconds": 0.2732156440000608, "nonpadding_tokens": 2979, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27049632299997484, "nonpadding_tokens": 3691, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2729348939999454, "nonpadding_tokens": 5821, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27337419499997395, "nonpadding_tokens": 4202, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27001090199996725, "nonpadding_tokens": 13578, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27241866199995, "nonpadding_tokens": 11595, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2735303480000084, "nonpadding_tokens": 19644, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27046634400005587, "nonpadding_tokens": 8371, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040451600000324106, "prefetch": {"data_wait_seconds": 0.011690854000633522, "consumed_batches": 1246, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1796, "tokens": 137400128, "loss_tokens": 48608, "loss": 2.236290997623848, "lr": 0.00029962993036227034, "optimizer_lrs": [0.019975328690818025, 0.00029962993036227034], "gradient_norm": 0.44587352871894836, "seconds": 0.2969898059999423, "nonpadding_tokens_per_second": 163668.91730960438, "ranks": [{"compute_seconds": 0.2919079600000032, "nonpadding_tokens": 3905, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29291467199993804, "nonpadding_tokens": 4047, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2898855490000187, "nonpadding_tokens": 8184, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2935112620000382, "nonpadding_tokens": 1597, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2940841869999531, "nonpadding_tokens": 20534, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2913020470000447, "nonpadding_tokens": 2850, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28898784900002283, "nonpadding_tokens": 4173, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29461800000001404, "nonpadding_tokens": 3318, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040855600002487336, "prefetch": {"data_wait_seconds": 0.01170102500061887, "consumed_batches": 1247, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1797, "tokens": 137452973, "loss_tokens": 52845, "loss": 2.282762974264358, "lr": 0.0002996295972828781, "optimizer_lrs": [0.01997530648552521, 0.0002996295972828781], "gradient_norm": 0.3728494942188263, "seconds": 0.32759265100003176, "nonpadding_tokens_per_second": 161313.14252222, "ranks": [{"compute_seconds": 0.3253311919999078, "nonpadding_tokens": 5950, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3249577249999902, "nonpadding_tokens": 10351, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3219114709999076, "nonpadding_tokens": 8591, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32445425700007036, "nonpadding_tokens": 10241, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3212647249999918, "nonpadding_tokens": 3682, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32348965500000304, "nonpadding_tokens": 9836, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32472073299993554, "nonpadding_tokens": 2371, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32152570200003083, "nonpadding_tokens": 1823, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035239500004990987, "prefetch": {"data_wait_seconds": 0.011709935000638438, "consumed_batches": 1248, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1798, "tokens": 137506591, "loss_tokens": 53618, "loss": 2.5490332071319335, "lr": 0.00029962925917838675, "optimizer_lrs": [0.019975283945225783, 0.00029962925917838675], "gradient_norm": 0.538611888885498, "seconds": 0.2563147600000093, "nonpadding_tokens_per_second": 209188.10918262394, "ranks": [{"compute_seconds": 0.25345589400001245, "nonpadding_tokens": 5234, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2540805300000102, "nonpadding_tokens": 7952, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25317614899995533, "nonpadding_tokens": 1833, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25366063900003155, "nonpadding_tokens": 10145, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2504798689999461, "nonpadding_tokens": 10458, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2518906509999397, "nonpadding_tokens": 11118, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25397730700001375, "nonpadding_tokens": 3165, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2506455069999447, "nonpadding_tokens": 3713, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037113700000190875, "prefetch": {"data_wait_seconds": 0.011719978000613196, "consumed_batches": 1249, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1799, "tokens": 137561957, "loss_tokens": 55366, "loss": 2.5762226027706534, "lr": 0.00029962890988971064, "optimizer_lrs": [0.019975260659314042, 0.00029962890988971064], "gradient_norm": 0.5005995035171509, "seconds": 0.2765153450000071, "nonpadding_tokens_per_second": 200227.58592293886, "ranks": [{"compute_seconds": 0.2726528819999885, "nonpadding_tokens": 3159, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27352452599996013, "nonpadding_tokens": 7740, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27271160499992675, "nonpadding_tokens": 9310, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2732870060000323, "nonpadding_tokens": 16547, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27402213200002734, "nonpadding_tokens": 1982, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27259260300002097, "nonpadding_tokens": 4047, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26994932299999164, "nonpadding_tokens": 2504, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2741661909999493, "nonpadding_tokens": 10077, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004182000000128028, "prefetch": {"data_wait_seconds": 0.011729217000564063, "consumed_batches": 1250, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1800, "tokens": 137632694, "loss_tokens": 70737, "loss": 2.4525069535745083, "lr": 0.00029962846339071813, "optimizer_lrs": [0.019975230892714545, 0.00029962846339071813], "gradient_norm": 0.47403788566589355, "seconds": 0.2778512749999891, "nonpadding_tokens_per_second": 254585.83913283385, "ranks": [{"compute_seconds": 0.27248876900000596, "nonpadding_tokens": 5697, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27134836799996265, "nonpadding_tokens": 2306, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27430690699998195, "nonpadding_tokens": 12767, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2746000600000116, "nonpadding_tokens": 11144, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2673828700000058, "nonpadding_tokens": 17461, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2699721159999626, "nonpadding_tokens": 6217, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2672072450000087, "nonpadding_tokens": 3272, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2754703210000571, "nonpadding_tokens": 11873, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035690999993676087, "prefetch": {"data_wait_seconds": 0.011738038000615347, "consumed_batches": 1251, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1801, "tokens": 137700992, "loss_tokens": 68298, "loss": 2.5146945646285395, "lr": 0.00029962803203255827, "optimizer_lrs": [0.019975202135503887, 0.00029962803203255827], "gradient_norm": 0.4937327802181244, "seconds": 0.2645268200000146, "nonpadding_tokens_per_second": 258189.3208408744, "ranks": [{"compute_seconds": 0.26083897199998773, "nonpadding_tokens": 1331, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26216444400006367, "nonpadding_tokens": 11579, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26088818800008085, "nonpadding_tokens": 10727, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2601494540000431, "nonpadding_tokens": 7035, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25814098600005764, "nonpadding_tokens": 9862, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26061463000007734, "nonpadding_tokens": 3754, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2620651600000201, "nonpadding_tokens": 5483, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25861094999993384, "nonpadding_tokens": 18527, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004502430000457025, "prefetch": {"data_wait_seconds": 0.011747573000619127, "consumed_batches": 1252, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1802, "tokens": 137786053, "loss_tokens": 85061, "loss": 2.4671022854186995, "lr": 0.0002996274944527868, "optimizer_lrs": [0.01997516629685245, 0.0002996274944527868], "gradient_norm": 0.3975253105163574, "seconds": 0.32646708600009333, "nonpadding_tokens_per_second": 260550.00227488685, "ranks": [{"compute_seconds": 0.3234279769999375, "nonpadding_tokens": 20287, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3202218010000024, "nonpadding_tokens": 9564, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3226363290000336, "nonpadding_tokens": 13731, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3210081710000168, "nonpadding_tokens": 4928, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3199241280000251, "nonpadding_tokens": 5027, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3208722030000217, "nonpadding_tokens": 7346, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3233848739999985, "nonpadding_tokens": 12248, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.324137114999985, "nonpadding_tokens": 11930, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000394863999986228, "prefetch": {"data_wait_seconds": 0.011757770000670575, "consumed_batches": 1253, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1803, "tokens": 137860405, "loss_tokens": 74352, "loss": 2.094020041491823, "lr": 0.0002996270242356588, "optimizer_lrs": [0.019975134949043922, 0.0002996270242356588], "gradient_norm": 0.47825905680656433, "seconds": 0.38725661399996625, "nonpadding_tokens_per_second": 191996.7208100582, "ranks": [{"compute_seconds": 0.38432867599999554, "nonpadding_tokens": 16870, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3810096600000179, "nonpadding_tokens": 19482, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3848019499999964, "nonpadding_tokens": 3853, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3843614310000021, "nonpadding_tokens": 5893, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3847481530000323, "nonpadding_tokens": 7436, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3814035119999062, "nonpadding_tokens": 5179, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37896512700001495, "nonpadding_tokens": 10987, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37974954699996033, "nonpadding_tokens": 4652, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040022899997893546, "prefetch": {"data_wait_seconds": 0.011766600000740368, "consumed_batches": 1254, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1804, "tokens": 137913139, "loss_tokens": 52734, "loss": 2.4859673076193727, "lr": 0.00029962669055561935, "optimizer_lrs": [0.01997511270370796, 0.00029962669055561935], "gradient_norm": 0.5423981547355652, "seconds": 0.3179519219999065, "nonpadding_tokens_per_second": 165855.26411762187, "ranks": [{"compute_seconds": 0.3109012539999867, "nonpadding_tokens": 5150, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31091432099992744, "nonpadding_tokens": 12100, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30972024799996234, "nonpadding_tokens": 3596, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3103158590000703, "nonpadding_tokens": 2816, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3155020379999769, "nonpadding_tokens": 2821, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3122213799999827, "nonpadding_tokens": 2908, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3145131460000812, "nonpadding_tokens": 10566, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3155540080000492, "nonpadding_tokens": 12777, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003905629999962912, "prefetch": {"data_wait_seconds": 0.011775143000775756, "consumed_batches": 1255, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1805, "tokens": 137976107, "loss_tokens": 62968, "loss": 2.724199940842968, "lr": 0.0002996262919236822, "optimizer_lrs": [0.019975086128245483, 0.0002996262919236822], "gradient_norm": 0.4879909157752991, "seconds": 0.30870894299994234, "nonpadding_tokens_per_second": 203972.06309637672, "ranks": [{"compute_seconds": 0.30609497500006455, "nonpadding_tokens": 14307, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30566018900003655, "nonpadding_tokens": 5703, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30255251199992017, "nonpadding_tokens": 3137, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3026099300000169, "nonpadding_tokens": 20474, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30153975600001104, "nonpadding_tokens": 3848, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30260356300004787, "nonpadding_tokens": 6162, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3053854190000038, "nonpadding_tokens": 4619, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3060513330000276, "nonpadding_tokens": 4718, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003614089999928183, "prefetch": {"data_wait_seconds": 0.011783233000826954, "consumed_batches": 1256, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1806, "tokens": 138024920, "loss_tokens": 48813, "loss": 2.372229541822875, "lr": 0.0002996259827567325, "optimizer_lrs": [0.019975065517115503, 0.0002996259827567325], "gradient_norm": 0.7159982323646545, "seconds": 0.24693567399992844, "nonpadding_tokens_per_second": 197674.96210375073, "ranks": [{"compute_seconds": 0.184247637999988, "nonpadding_tokens": 3881, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.18114907100004984, "nonpadding_tokens": 2352, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.1803852520000646, "nonpadding_tokens": 12275, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.18476306299999123, "nonpadding_tokens": 11621, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.18002157599994462, "nonpadding_tokens": 8918, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.18239398799994433, "nonpadding_tokens": 3537, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.18461689000002934, "nonpadding_tokens": 3418, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.18074208899997757, "nonpadding_tokens": 2811, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.033233293999956004, "prefetch": {"data_wait_seconds": 0.011792991000859843, "consumed_batches": 1257, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1807, "tokens": 138095123, "loss_tokens": 70203, "loss": 2.5814192413429624, "lr": 0.00029962553788809613, "optimizer_lrs": [0.01997503585920641, 0.00029962553788809613], "gradient_norm": 0.3625580966472626, "seconds": 0.26354184900003474, "nonpadding_tokens_per_second": 266382.7406021984, "ranks": [{"compute_seconds": 0.2586795480000319, "nonpadding_tokens": 14120, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2572976530000233, "nonpadding_tokens": 9733, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25950456599991867, "nonpadding_tokens": 7837, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25791983600004187, "nonpadding_tokens": 7963, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26053348300001744, "nonpadding_tokens": 7918, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25777684000001955, "nonpadding_tokens": 6177, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25632702799998697, "nonpadding_tokens": 10945, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2611624839998967, "nonpadding_tokens": 5510, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003973540000288267, "prefetch": {"data_wait_seconds": 0.011802382000908551, "consumed_batches": 1258, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1808, "tokens": 138168009, "loss_tokens": 72886, "loss": 2.4330615618911726, "lr": 0.00029962507573821017, "optimizer_lrs": [0.019975005049214012, 0.00029962507573821017], "gradient_norm": 0.845237672328949, "seconds": 0.28437339699996755, "nonpadding_tokens_per_second": 256303.8623476032, "ranks": [{"compute_seconds": 0.2816307589999951, "nonpadding_tokens": 5374, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28150951500003885, "nonpadding_tokens": 1150, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27826272300001165, "nonpadding_tokens": 12649, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28129793300001893, "nonpadding_tokens": 16077, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28187957800003005, "nonpadding_tokens": 11232, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2803980070000307, "nonpadding_tokens": 12534, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27727969700003996, "nonpadding_tokens": 8774, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28198709499997676, "nonpadding_tokens": 5096, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003814670000110709, "prefetch": {"data_wait_seconds": 0.01181078800091484, "consumed_batches": 1259, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1809, "tokens": 138239339, "loss_tokens": 71330, "loss": 2.2590775269872423, "lr": 0.00029962462317892385, "optimizer_lrs": [0.019974974878594925, 0.00029962462317892385], "gradient_norm": 0.3966399133205414, "seconds": 0.3292835050000349, "nonpadding_tokens_per_second": 216621.84384241307, "ranks": [{"compute_seconds": 0.32522463299994797, "nonpadding_tokens": 7473, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32602379100001144, "nonpadding_tokens": 5730, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3226211780000767, "nonpadding_tokens": 11574, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32567788300002576, "nonpadding_tokens": 13427, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3263042949999999, "nonpadding_tokens": 12911, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3249185939999961, "nonpadding_tokens": 4538, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3219419920000064, "nonpadding_tokens": 5379, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32685725700002877, "nonpadding_tokens": 10298, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003906610000967703, "prefetch": {"data_wait_seconds": 0.011820188000982057, "consumed_batches": 1260, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1810, "tokens": 138307031, "loss_tokens": 67692, "loss": 2.179829457321397, "lr": 0.00029962419344914924, "optimizer_lrs": [0.019974946229943284, 0.00029962419344914924], "gradient_norm": 0.5211228728294373, "seconds": 0.3150001160000784, "nonpadding_tokens_per_second": 214895.15895918958, "ranks": [{"compute_seconds": 0.3121518570000035, "nonpadding_tokens": 14234, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3112923989999672, "nonpadding_tokens": 3554, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3085729110000557, "nonpadding_tokens": 13779, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30874535399993874, "nonpadding_tokens": 6035, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3126167839999425, "nonpadding_tokens": 3412, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3096376880000662, "nonpadding_tokens": 1514, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31201513700000305, "nonpadding_tokens": 10058, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31259428700002445, "nonpadding_tokens": 15106, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035539399993922416, "prefetch": {"data_wait_seconds": 0.011827939000980336, "consumed_batches": 1261, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1811, "tokens": 138369592, "loss_tokens": 62561, "loss": 2.7282682102268185, "lr": 0.00029962379607426543, "optimizer_lrs": [0.019974919738284365, 0.00029962379607426543], "gradient_norm": 0.39761772751808167, "seconds": 0.22446592900007545, "nonpadding_tokens_per_second": 278710.44963790016, "ranks": [{"compute_seconds": 0.2217731590000085, "nonpadding_tokens": 3560, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22207229499997538, "nonpadding_tokens": 7219, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21915064600000278, "nonpadding_tokens": 9615, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22192821999999524, "nonpadding_tokens": 4607, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21811159999992924, "nonpadding_tokens": 13056, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2200347259999944, "nonpadding_tokens": 16790, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2221656909999865, "nonpadding_tokens": 6080, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2191708479999761, "nonpadding_tokens": 1634, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000399503999915396, "prefetch": {"data_wait_seconds": 0.011837068000957629, "consumed_batches": 1262, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1812, "tokens": 138433470, "loss_tokens": 63878, "loss": 1.9740525885281317, "lr": 0.00029962339011770843, "optimizer_lrs": [0.019974892674513897, 0.00029962339011770843], "gradient_norm": 0.7238043546676636, "seconds": 0.2984079970000266, "nonpadding_tokens_per_second": 214062.62781889958, "ranks": [{"compute_seconds": 0.23473206900007426, "nonpadding_tokens": 1911, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2355407860000014, "nonpadding_tokens": 11021, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2380472730000065, "nonpadding_tokens": 9936, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23482056299997112, "nonpadding_tokens": 7789, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23320906499998273, "nonpadding_tokens": 3435, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23627220399998805, "nonpadding_tokens": 7629, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23871836500006793, "nonpadding_tokens": 16106, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23937358399996356, "nonpadding_tokens": 6051, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.017729048999967745, "prefetch": {"data_wait_seconds": 0.011845046000942716, "consumed_batches": 1263, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1813, "tokens": 138511196, "loss_tokens": 77726, "loss": 2.407041643079536, "lr": 0.00029962289585970764, "optimizer_lrs": [0.019974859723980512, 0.00029962289585970764], "gradient_norm": 0.40876254439353943, "seconds": 0.37748758799989446, "nonpadding_tokens_per_second": 205903.4587384148, "ranks": [{"compute_seconds": 0.3742871890000288, "nonpadding_tokens": 11813, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37446711899997354, "nonpadding_tokens": 22847, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3738170320000336, "nonpadding_tokens": 3373, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.37441693400000986, "nonpadding_tokens": 12078, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.37510065200001463, "nonpadding_tokens": 5643, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3732137370000146, "nonpadding_tokens": 5346, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37032649900004344, "nonpadding_tokens": 7223, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37502239399998416, "nonpadding_tokens": 9403, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003855909999401774, "prefetch": {"data_wait_seconds": 0.011854807000943879, "consumed_batches": 1264, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1814, "tokens": 138570592, "loss_tokens": 59396, "loss": 2.618890581857364, "lr": 0.00029962251794362206, "optimizer_lrs": [0.019974834529574804, 0.00029962251794362206], "gradient_norm": 0.41024965047836304, "seconds": 0.3017274339999858, "nonpadding_tokens_per_second": 196853.16383926425, "ranks": [{"compute_seconds": 0.2994278439999789, "nonpadding_tokens": 7206, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2953405569999177, "nonpadding_tokens": 3554, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29807339200010574, "nonpadding_tokens": 2685, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29865964800001166, "nonpadding_tokens": 9027, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2954984149999973, "nonpadding_tokens": 7361, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.297959909000042, "nonpadding_tokens": 9715, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29897741600007066, "nonpadding_tokens": 3875, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29584872799989625, "nonpadding_tokens": 15973, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039831600008710666, "prefetch": {"data_wait_seconds": 0.011865036000884288, "consumed_batches": 1265, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1815, "tokens": 138602741, "loss_tokens": 32149, "loss": 2.4404642251703006, "lr": 0.0002996223133118817, "optimizer_lrs": [0.01997482088745878, 0.0002996223133118817], "gradient_norm": 0.4436753988265991, "seconds": 0.23344063499996537, "nonpadding_tokens_per_second": 137718.09693717107, "ranks": [{"compute_seconds": 0.23041267899998275, "nonpadding_tokens": 1124, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23111583000002156, "nonpadding_tokens": 2826, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22786648699991474, "nonpadding_tokens": 3859, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2297672990000592, "nonpadding_tokens": 4363, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22665940499996395, "nonpadding_tokens": 7967, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22946177599999373, "nonpadding_tokens": 3691, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22720820600000025, "nonpadding_tokens": 3565, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23043636599993533, "nonpadding_tokens": 4754, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003734970000550675, "prefetch": {"data_wait_seconds": 0.011874253000954127, "consumed_batches": 1266, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1816, "tokens": 138693019, "loss_tokens": 90278, "loss": 3.496681777398702, "lr": 0.00029962173838692465, "optimizer_lrs": [0.019974782559128313, 0.00029962173838692465], "gradient_norm": 0.818568229675293, "seconds": 0.30665859999999157, "nonpadding_tokens_per_second": 294392.5264121159, "ranks": [{"compute_seconds": 0.2993348780000815, "nonpadding_tokens": 3035, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29968266200000926, "nonpadding_tokens": 15467, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2985962540000173, "nonpadding_tokens": 19858, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2994998190000615, "nonpadding_tokens": 18565, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3040783400000464, "nonpadding_tokens": 25355, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30056072100001074, "nonpadding_tokens": 2821, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3033645250000063, "nonpadding_tokens": 2658, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3042188089999627, "nonpadding_tokens": 2519, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037712599998940277, "prefetch": {"data_wait_seconds": 0.011884122000950015, "consumed_batches": 1267, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1817, "tokens": 138790362, "loss_tokens": 97343, "loss": 2.4027799893161297, "lr": 0.00029962111798010656, "optimizer_lrs": [0.019974741198673773, 0.00029962111798010656], "gradient_norm": 0.482646107673645, "seconds": 0.41472456200006036, "nonpadding_tokens_per_second": 234717.22902195947, "ranks": [{"compute_seconds": 0.4107133120000981, "nonpadding_tokens": 4584, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.41236011199998757, "nonpadding_tokens": 11251, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.41136225200000354, "nonpadding_tokens": 9648, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4122334900000624, "nonpadding_tokens": 4423, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4083082240000522, "nonpadding_tokens": 4603, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.409317150999982, "nonpadding_tokens": 24083, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.40795205399990664, "nonpadding_tokens": 21620, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4088821640000333, "nonpadding_tokens": 17131, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040198499993948644, "prefetch": {"data_wait_seconds": 0.011893539000951137, "consumed_batches": 1268, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1818, "tokens": 138863237, "loss_tokens": 72875, "loss": 2.2313481989708404, "lr": 0.0002996206531855931, "optimizer_lrs": [0.019974710212372872, 0.0002996206531855931], "gradient_norm": 0.9752598404884338, "seconds": 0.2437497280000116, "nonpadding_tokens_per_second": 298974.6925994376, "ranks": [{"compute_seconds": 0.2408231250000199, "nonpadding_tokens": 2085, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23751181300008284, "nonpadding_tokens": 3526, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23817386100006388, "nonpadding_tokens": 16408, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24045080500002314, "nonpadding_tokens": 5090, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2411670139999842, "nonpadding_tokens": 1747, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23996262599996498, "nonpadding_tokens": 14205, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2369820140000911, "nonpadding_tokens": 12805, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23764852199997222, "nonpadding_tokens": 17009, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004212640000105239, "prefetch": {"data_wait_seconds": 0.011903131000920553, "consumed_batches": 1269, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1819, "tokens": 138910717, "loss_tokens": 47480, "loss": 2.5032227385214827, "lr": 0.00029962035020653755, "optimizer_lrs": [0.019974690013769173, 0.00029962035020653755], "gradient_norm": 0.4860233962535858, "seconds": 0.25617161799993937, "nonpadding_tokens_per_second": 185344.49823403635, "ranks": [{"compute_seconds": 0.25261617000001024, "nonpadding_tokens": 10425, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2532548039999938, "nonpadding_tokens": 13698, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24978266999994503, "nonpadding_tokens": 3538, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2506267809999372, "nonpadding_tokens": 3328, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2494005969999762, "nonpadding_tokens": 1872, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2517422800000304, "nonpadding_tokens": 2880, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25298706800003856, "nonpadding_tokens": 4786, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2538846459999604, "nonpadding_tokens": 6953, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037113600001248415, "prefetch": {"data_wait_seconds": 0.011910818000913537, "consumed_batches": 1270, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1820, "tokens": 139001137, "loss_tokens": 90420, "loss": 2.681872788099978, "lr": 0.0002996197728850409, "optimizer_lrs": [0.019974651525669393, 0.0002996197728850409], "gradient_norm": 0.34076932072639465, "seconds": 0.31809959399993204, "nonpadding_tokens_per_second": 284250.59857202874, "ranks": [{"compute_seconds": 0.31515427499994075, "nonpadding_tokens": 12776, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31515413699992223, "nonpadding_tokens": 20821, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31229654100002335, "nonpadding_tokens": 4157, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31302096999991136, "nonpadding_tokens": 12280, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3157657979999158, "nonpadding_tokens": 13794, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3137682159999713, "nonpadding_tokens": 4362, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.315698655999995, "nonpadding_tokens": 10577, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3156169490000593, "nonpadding_tokens": 11653, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003679739999142839, "prefetch": {"data_wait_seconds": 0.011920474001001935, "consumed_batches": 1271, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1821, "tokens": 139110278, "loss_tokens": 109141, "loss": 2.0730641727215255, "lr": 0.00029961907544864906, "optimizer_lrs": [0.01997460502990994, 0.00029961907544864906], "gradient_norm": 0.41732165217399597, "seconds": 0.26518819699992946, "nonpadding_tokens_per_second": 411560.5492050954, "ranks": [{"compute_seconds": 0.26285767600006693, "nonpadding_tokens": 20938, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26277207199996155, "nonpadding_tokens": 21985, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2617270590000089, "nonpadding_tokens": 14916, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2600740789999918, "nonpadding_tokens": 6447, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26257567400000426, "nonpadding_tokens": 5539, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2611830059998965, "nonpadding_tokens": 6518, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.262211314999945, "nonpadding_tokens": 12531, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2628163660000382, "nonpadding_tokens": 20267, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035794899997654284, "prefetch": {"data_wait_seconds": 0.011929649000947506, "consumed_batches": 1272, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1822, "tokens": 139218094, "loss_tokens": 107816, "loss": 2.522971138328263, "lr": 0.00029961838585273846, "optimizer_lrs": [0.019974559056849233, 0.00029961838585273846], "gradient_norm": 0.43575069308280945, "seconds": 0.36896323600001324, "nonpadding_tokens_per_second": 292213.3954831102, "ranks": [{"compute_seconds": 0.3654199759999983, "nonpadding_tokens": 18529, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3631987179999214, "nonpadding_tokens": 12182, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36338295899997775, "nonpadding_tokens": 16809, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3666461349999963, "nonpadding_tokens": 26916, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36652010800003154, "nonpadding_tokens": 23463, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36528701499992167, "nonpadding_tokens": 3244, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36626830500006236, "nonpadding_tokens": 3929, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36328022600002896, "nonpadding_tokens": 2744, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003509020000365126, "prefetch": {"data_wait_seconds": 0.011938029001001382, "consumed_batches": 1273, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1823, "tokens": 139259376, "loss_tokens": 41282, "loss": 2.646303360120634, "lr": 0.0002996181216463657, "optimizer_lrs": [0.01997454144309105, 0.0002996181216463657], "gradient_norm": 0.44376951456069946, "seconds": 0.25446165100004237, "nonpadding_tokens_per_second": 162232.69729548806, "ranks": [{"compute_seconds": 0.2509507310001027, "nonpadding_tokens": 5480, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.248168312999951, "nonpadding_tokens": 4998, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25086317800003144, "nonpadding_tokens": 3659, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24949470599995038, "nonpadding_tokens": 5193, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25217489800002113, "nonpadding_tokens": 3772, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24902650000001358, "nonpadding_tokens": 3201, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24807208400000036, "nonpadding_tokens": 4291, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24837605899995197, "nonpadding_tokens": 10688, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003784300000688745, "prefetch": {"data_wait_seconds": 0.01194600000094681, "consumed_batches": 1274, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1824, "tokens": 139312079, "loss_tokens": 52703, "loss": 2.3955058298389087, "lr": 0.0002996177842124769, "optimizer_lrs": [0.019974518947498462, 0.0002996177842124769], "gradient_norm": 0.42552846670150757, "seconds": 0.26782371399997373, "nonpadding_tokens_per_second": 196782.42532326755, "ranks": [{"compute_seconds": 0.2624724170000263, "nonpadding_tokens": 3173, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26435060000005706, "nonpadding_tokens": 4350, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2612644230000569, "nonpadding_tokens": 2383, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2616871060000676, "nonpadding_tokens": 7132, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26474894000000404, "nonpadding_tokens": 6408, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26157251900008305, "nonpadding_tokens": 15727, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2602552140000398, "nonpadding_tokens": 5743, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26542995600004815, "nonpadding_tokens": 7787, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003802989999712736, "prefetch": {"data_wait_seconds": 0.011955006000974322, "consumed_batches": 1275, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1825, "tokens": 139351464, "loss_tokens": 39385, "loss": 2.7256091706868095, "lr": 0.0002996175319506728, "optimizer_lrs": [0.019974502130044855, 0.0002996175319506728], "gradient_norm": 0.4596789479255676, "seconds": 0.24535478500001773, "nonpadding_tokens_per_second": 160522.6488653855, "ranks": [{"compute_seconds": 0.24098198499996215, "nonpadding_tokens": 4268, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24235840000005737, "nonpadding_tokens": 2541, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24214020299996264, "nonpadding_tokens": 8495, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23998362300005738, "nonpadding_tokens": 10450, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24284602399995947, "nonpadding_tokens": 1740, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24166739000008874, "nonpadding_tokens": 3699, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23805954399995244, "nonpadding_tokens": 2554, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24296658699995533, "nonpadding_tokens": 5638, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003824909999821102, "prefetch": {"data_wait_seconds": 0.01196468000098605, "consumed_batches": 1276, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1826, "tokens": 139409154, "loss_tokens": 57690, "loss": 2.4323637112151153, "lr": 0.000299617162294925, "optimizer_lrs": [0.019974477486328336, 0.000299617162294925], "gradient_norm": 0.39191505312919617, "seconds": 0.22160733999999138, "nonpadding_tokens_per_second": 260325.31232946637, "ranks": [{"compute_seconds": 0.2179748200001086, "nonpadding_tokens": 5833, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.215261658999907, "nonpadding_tokens": 5751, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21774450700002035, "nonpadding_tokens": 4255, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2162513889999218, "nonpadding_tokens": 3977, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21454000199992151, "nonpadding_tokens": 8641, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2171441190000678, "nonpadding_tokens": 10682, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2189696579999918, "nonpadding_tokens": 2563, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21912860700001602, "nonpadding_tokens": 15988, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003867760000275666, "prefetch": {"data_wait_seconds": 0.011972890001018754, "consumed_batches": 1277, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1827, "tokens": 139460527, "loss_tokens": 51373, "loss": 2.6251113182021686, "lr": 0.00029961683296604884, "optimizer_lrs": [0.019974455531069923, 0.00029961683296604884], "gradient_norm": 0.5296806693077087, "seconds": 0.24464389499996742, "nonpadding_tokens_per_second": 209990.93396549643, "ranks": [{"compute_seconds": 0.24111512199999652, "nonpadding_tokens": 4523, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2413982439999245, "nonpadding_tokens": 1651, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23836695199997848, "nonpadding_tokens": 9555, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23886474799996904, "nonpadding_tokens": 5414, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24174581700003728, "nonpadding_tokens": 8012, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2387882669999044, "nonpadding_tokens": 12042, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.240906128000006, "nonpadding_tokens": 5901, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24221017600007144, "nonpadding_tokens": 4275, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000423139999952582, "prefetch": {"data_wait_seconds": 0.01198259800101198, "consumed_batches": 1278, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1828, "tokens": 139513935, "loss_tokens": 53408, "loss": 2.489610346296435, "lr": 0.0002996164904418165, "optimizer_lrs": [0.019974432696121102, 0.0002996164904418165], "gradient_norm": 0.5847187638282776, "seconds": 0.2697445850000122, "nonpadding_tokens_per_second": 197994.70673339962, "ranks": [{"compute_seconds": 0.2665436199999931, "nonpadding_tokens": 2901, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26400372899991, "nonpadding_tokens": 10789, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2662521920000245, "nonpadding_tokens": 10234, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2667156840000189, "nonpadding_tokens": 8339, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2628652030000467, "nonpadding_tokens": 4290, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26613120700005766, "nonpadding_tokens": 5031, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2673436180000408, "nonpadding_tokens": 8758, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26327039700004207, "nonpadding_tokens": 3066, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004234899998891706, "prefetch": {"data_wait_seconds": 0.011992433000912683, "consumed_batches": 1279, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1829, "tokens": 139569138, "loss_tokens": 55203, "loss": 2.3794562795500243, "lr": 0.0002996161362450196, "optimizer_lrs": [0.01997440908300131, 0.0002996161362450196], "gradient_norm": 0.4313027560710907, "seconds": 0.3154125970000905, "nonpadding_tokens_per_second": 175018.37442460854, "ranks": [{"compute_seconds": 0.31116672400003154, "nonpadding_tokens": 8929, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31066359300007207, "nonpadding_tokens": 9941, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31042198700004064, "nonpadding_tokens": 5280, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3080691070000512, "nonpadding_tokens": 10714, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3063846800000647, "nonpadding_tokens": 4604, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30893510900000365, "nonpadding_tokens": 6931, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31119580199992924, "nonpadding_tokens": 6149, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3129202669999813, "nonpadding_tokens": 2655, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040553600001658197, "prefetch": {"data_wait_seconds": 0.012001814000953459, "consumed_batches": 1280, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1830, "tokens": 139628756, "loss_tokens": 59618, "loss": 2.9300892767285047, "lr": 0.0002996157535370741, "optimizer_lrs": [0.019974383569138273, 0.0002996157535370741], "gradient_norm": 0.48661279678344727, "seconds": 0.2695441529999698, "nonpadding_tokens_per_second": 221180.83192109413, "ranks": [{"compute_seconds": 0.26659311599996727, "nonpadding_tokens": 2201, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.267030415000022, "nonpadding_tokens": 3363, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2640357950000407, "nonpadding_tokens": 10548, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26439711099999386, "nonpadding_tokens": 4969, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2672484560000612, "nonpadding_tokens": 9916, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2659535489999598, "nonpadding_tokens": 1659, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2667824440000004, "nonpadding_tokens": 9947, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2635670490000166, "nonpadding_tokens": 17015, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035060300001532596, "prefetch": {"data_wait_seconds": 0.012010608000949219, "consumed_batches": 1281, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1831, "tokens": 139667089, "loss_tokens": 38333, "loss": 2.6202615077087628, "lr": 0.0002996155073641106, "optimizer_lrs": [0.019974367157607375, 0.0002996155073641106], "gradient_norm": 0.4398757219314575, "seconds": 0.2716216359999635, "nonpadding_tokens_per_second": 141126.4601911357, "ranks": [{"compute_seconds": 0.2683422039999641, "nonpadding_tokens": 6701, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26620780500002184, "nonpadding_tokens": 8677, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2681970559999627, "nonpadding_tokens": 5267, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2663010700000541, "nonpadding_tokens": 3765, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26921386900005473, "nonpadding_tokens": 3239, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26673565299995516, "nonpadding_tokens": 2539, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2642149650000647, "nonpadding_tokens": 3556, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2652979360000245, "nonpadding_tokens": 4589, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003336690000423914, "prefetch": {"data_wait_seconds": 0.01201871200085236, "consumed_batches": 1282, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1832, "tokens": 139715009, "loss_tokens": 47920, "loss": 2.634269681239566, "lr": 0.0002996151995131113, "optimizer_lrs": [0.019974346634207423, 0.0002996151995131113], "gradient_norm": 0.4359053373336792, "seconds": 0.3948276729998952, "nonpadding_tokens_per_second": 121369.40563437335, "ranks": [{"compute_seconds": 0.3900399290000678, "nonpadding_tokens": 7652, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.38859065700000883, "nonpadding_tokens": 2225, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3889806789999284, "nonpadding_tokens": 6262, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.38979645300003085, "nonpadding_tokens": 2498, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3886245220001001, "nonpadding_tokens": 11738, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38947474200006127, "nonpadding_tokens": 2917, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3923777909999444, "nonpadding_tokens": 11283, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3925388000000112, "nonpadding_tokens": 3345, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034046700000089913, "prefetch": {"data_wait_seconds": 0.012027915000956568, "consumed_batches": 1283, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1833, "tokens": 139801212, "loss_tokens": 86203, "loss": 2.44577316769718, "lr": 0.00029961464541210643, "optimizer_lrs": [0.019974309694140432, 0.00029961464541210643], "gradient_norm": 0.43987026810646057, "seconds": 0.2528980590000174, "nonpadding_tokens_per_second": 340860.66275421303, "ranks": [{"compute_seconds": 0.2504357840000466, "nonpadding_tokens": 2729, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24961840899993604, "nonpadding_tokens": 6058, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24691937100010364, "nonpadding_tokens": 6406, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25018483300004846, "nonpadding_tokens": 15264, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24565351900002952, "nonpadding_tokens": 7652, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24819741299995712, "nonpadding_tokens": 14765, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2503378149999662, "nonpadding_tokens": 17525, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24683967699991172, "nonpadding_tokens": 15804, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034585999992486904, "prefetch": {"data_wait_seconds": 0.012036891000960281, "consumed_batches": 1284, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1834, "tokens": 139870533, "loss_tokens": 69321, "loss": 2.5429895432120135, "lr": 0.0002996141995374921, "optimizer_lrs": [0.01997427996916614, 0.0002996141995374921], "gradient_norm": 0.3971104621887207, "seconds": 0.2827047119999406, "nonpadding_tokens_per_second": 245206.3834012592, "ranks": [{"compute_seconds": 0.2783648259999154, "nonpadding_tokens": 8949, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27881695300004594, "nonpadding_tokens": 4511, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2778430749999643, "nonpadding_tokens": 18486, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27647448799996255, "nonpadding_tokens": 9056, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28023528400001396, "nonpadding_tokens": 14599, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27730601600001137, "nonpadding_tokens": 7661, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27835998499995185, "nonpadding_tokens": 2920, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27533779800000957, "nonpadding_tokens": 3139, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044298499994965823, "prefetch": {"data_wait_seconds": 0.012046518000900619, "consumed_batches": 1285, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1835, "tokens": 139939085, "loss_tokens": 68552, "loss": 2.433697229840121, "lr": 0.00029961375835594364, "optimizer_lrs": [0.019974250557062912, 0.00029961375835594364], "gradient_norm": 0.5172249674797058, "seconds": 0.27086115399993105, "nonpadding_tokens_per_second": 253089.078989959, "ranks": [{"compute_seconds": 0.26616498899988983, "nonpadding_tokens": 3292, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26853975500000615, "nonpadding_tokens": 18958, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26548185800004376, "nonpadding_tokens": 12179, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2679706679999754, "nonpadding_tokens": 15863, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2644526249999899, "nonpadding_tokens": 2949, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26575149600000714, "nonpadding_tokens": 5215, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26483499500000107, "nonpadding_tokens": 4303, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2650722890000452, "nonpadding_tokens": 5793, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004038029999264836, "prefetch": {"data_wait_seconds": 0.012056799000973228, "consumed_batches": 1286, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1836, "tokens": 140028285, "loss_tokens": 89200, "loss": 2.625724320347534, "lr": 0.00029961318391271514, "optimizer_lrs": [0.01997421226084768, 0.00029961318391271514], "gradient_norm": 0.4475776255130768, "seconds": 0.2635133759999917, "nonpadding_tokens_per_second": 338502.7407489281, "ranks": [{"compute_seconds": 0.25858338800003366, "nonpadding_tokens": 3298, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2608185799999774, "nonpadding_tokens": 10608, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2602655009999353, "nonpadding_tokens": 11155, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2592221620000146, "nonpadding_tokens": 5924, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2573038970000425, "nonpadding_tokens": 11494, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26010582100002466, "nonpadding_tokens": 17603, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2611204819999102, "nonpadding_tokens": 9726, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2582557410000845, "nonpadding_tokens": 19392, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039171999992504425, "prefetch": {"data_wait_seconds": 0.01206512600090548, "consumed_batches": 1287, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1837, "tokens": 140109239, "loss_tokens": 80954, "loss": 2.3910805055957702, "lr": 0.00029961266220432565, "optimizer_lrs": [0.019974177480288378, 0.00029961266220432565], "gradient_norm": 0.49219340085983276, "seconds": 0.2945821819999992, "nonpadding_tokens_per_second": 274809.560613548, "ranks": [{"compute_seconds": 0.2900887729999795, "nonpadding_tokens": 21160, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2883806019999611, "nonpadding_tokens": 14294, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2886842750000369, "nonpadding_tokens": 21327, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28926442800002405, "nonpadding_tokens": 3455, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2878988819999222, "nonpadding_tokens": 4245, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28895241000009264, "nonpadding_tokens": 5194, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28746923999995033, "nonpadding_tokens": 4417, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.292275539000002, "nonpadding_tokens": 6862, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033846600001652405, "prefetch": {"data_wait_seconds": 0.012073843000919737, "consumed_batches": 1288, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1838, "tokens": 140170355, "loss_tokens": 61116, "loss": 2.5469327794685515, "lr": 0.0002996122681094392, "optimizer_lrs": [0.01997415120729595, 0.0002996122681094392], "gradient_norm": 0.4825780391693115, "seconds": 0.3091092709998975, "nonpadding_tokens_per_second": 197716.48971350415, "ranks": [{"compute_seconds": 0.30584482999995544, "nonpadding_tokens": 14770, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30318655700000363, "nonpadding_tokens": 5650, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3038615909999862, "nonpadding_tokens": 2257, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30624550000004547, "nonpadding_tokens": 11100, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3027801130000398, "nonpadding_tokens": 4914, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30528939399994215, "nonpadding_tokens": 5211, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3067858839999644, "nonpadding_tokens": 7112, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3033381569999847, "nonpadding_tokens": 10102, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036930400005985575, "prefetch": {"data_wait_seconds": 0.012082947000934041, "consumed_batches": 1289, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1839, "tokens": 140241302, "loss_tokens": 70947, "loss": 2.3875507421032602, "lr": 0.00029961181037025796, "optimizer_lrs": [0.01997412069135053, 0.00029961181037025796], "gradient_norm": 0.3872503340244293, "seconds": 0.303487658999984, "nonpadding_tokens_per_second": 233772.2734221748, "ranks": [{"compute_seconds": 0.29846780500008663, "nonpadding_tokens": 12540, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29700914099998954, "nonpadding_tokens": 3296, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29749697099998684, "nonpadding_tokens": 6249, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29878938100000596, "nonpadding_tokens": 9154, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3005886160000273, "nonpadding_tokens": 12807, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29773927799999456, "nonpadding_tokens": 11621, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29675943400002325, "nonpadding_tokens": 10593, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3011059000000387, "nonpadding_tokens": 4687, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003822620000164534, "prefetch": {"data_wait_seconds": 0.012102877000941703, "consumed_batches": 1290, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1840, "tokens": 140291857, "loss_tokens": 50555, "loss": 2.4306786853426963, "lr": 0.00029961148403260965, "optimizer_lrs": [0.01997409893550731, 0.00029961148403260965], "gradient_norm": 0.6157883405685425, "seconds": 0.20727902600003745, "nonpadding_tokens_per_second": 243898.29002762135, "ranks": [{"compute_seconds": 0.20298294700000952, "nonpadding_tokens": 2694, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20495777099995394, "nonpadding_tokens": 17843, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20390121399998407, "nonpadding_tokens": 8226, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.20197465000001102, "nonpadding_tokens": 9791, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20113426099999288, "nonpadding_tokens": 2210, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2019269400000212, "nonpadding_tokens": 1295, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20040365700003804, "nonpadding_tokens": 4765, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.201900261999981, "nonpadding_tokens": 3731, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035070699993866583, "prefetch": {"data_wait_seconds": 0.012111340001027315, "consumed_batches": 1291, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1841, "tokens": 140347020, "loss_tokens": 55163, "loss": 2.1016445012961587, "lr": 0.0002996111277936511, "optimizer_lrs": [0.01997407518624341, 0.0002996111277936511], "gradient_norm": 0.4142720699310303, "seconds": 0.2750911340000357, "nonpadding_tokens_per_second": 200526.27359481837, "ranks": [{"compute_seconds": 0.2699581000000535, "nonpadding_tokens": 9617, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26944249399991804, "nonpadding_tokens": 5112, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2675994329999867, "nonpadding_tokens": 2651, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2721789180000087, "nonpadding_tokens": 5196, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27272227200001, "nonpadding_tokens": 8148, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26976353500003825, "nonpadding_tokens": 4388, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2723120610000933, "nonpadding_tokens": 11566, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2678656109999338, "nonpadding_tokens": 8485, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004355800000439558, "prefetch": {"data_wait_seconds": 0.012121700001060276, "consumed_batches": 1292, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1842, "tokens": 140408592, "loss_tokens": 61572, "loss": 2.5928389730721757, "lr": 0.0002996107299732798, "optimizer_lrs": [0.01997404866488532, 0.0002996107299732798], "gradient_norm": 0.5431306958198547, "seconds": 0.34622031799995057, "nonpadding_tokens_per_second": 177840.51599192625, "ranks": [{"compute_seconds": 0.3423529140000028, "nonpadding_tokens": 14922, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34014578899996195, "nonpadding_tokens": 12145, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3395309630000156, "nonpadding_tokens": 9590, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34023242500006745, "nonpadding_tokens": 4396, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3438424189999978, "nonpadding_tokens": 3352, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3405586860000085, "nonpadding_tokens": 4007, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3429690010000286, "nonpadding_tokens": 3582, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3391925610000044, "nonpadding_tokens": 9578, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041741899997305154, "prefetch": {"data_wait_seconds": 0.012132469001087998, "consumed_batches": 1293, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1843, "tokens": 140452433, "loss_tokens": 43841, "loss": 2.2925324254122854, "lr": 0.00029961044659017474, "optimizer_lrs": [0.019974029772678318, 0.00029961044659017474], "gradient_norm": 0.438870370388031, "seconds": 0.28900342300005377, "nonpadding_tokens_per_second": 151697.16519237158, "ranks": [{"compute_seconds": 0.2837226340000143, "nonpadding_tokens": 3186, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2865458910000598, "nonpadding_tokens": 2763, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2835295559999622, "nonpadding_tokens": 760, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2861690130000625, "nonpadding_tokens": 10868, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2866631150000103, "nonpadding_tokens": 3140, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28376819300001443, "nonpadding_tokens": 7506, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2862264980000191, "nonpadding_tokens": 3590, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2829920950000542, "nonpadding_tokens": 12028, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004014400000187379, "prefetch": {"data_wait_seconds": 0.012141909001073259, "consumed_batches": 1294, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1844, "tokens": 140482128, "loss_tokens": 29695, "loss": 2.6704814046977607, "lr": 0.0002996102545866731, "optimizer_lrs": [0.019974016972444875, 0.0002996102545866731], "gradient_norm": 0.41826412081718445, "seconds": 0.24748957100007374, "nonpadding_tokens_per_second": 119984.85382639074, "ranks": [{"compute_seconds": 0.2435122169999886, "nonpadding_tokens": 8359, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24349209399997562, "nonpadding_tokens": 2630, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24294593899992378, "nonpadding_tokens": 5351, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24350804400000925, "nonpadding_tokens": 2928, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2450174320000542, "nonpadding_tokens": 1617, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24249130199996216, "nonpadding_tokens": 2093, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24379147599995576, "nonpadding_tokens": 3425, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24009343500006253, "nonpadding_tokens": 3292, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038533600002210733, "prefetch": {"data_wait_seconds": 0.01215013000110332, "consumed_batches": 1295, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1845, "tokens": 140530208, "loss_tokens": 48080, "loss": 2.53799803062604, "lr": 0.0002996099436083062, "optimizer_lrs": [0.019973996240553748, 0.0002996099436083062], "gradient_norm": 0.6484583616256714, "seconds": 0.2922593250000318, "nonpadding_tokens_per_second": 164511.43175669335, "ranks": [{"compute_seconds": 0.2897124999999505, "nonpadding_tokens": 5513, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28931324099994526, "nonpadding_tokens": 1634, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28855504000000565, "nonpadding_tokens": 4131, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28904698899998493, "nonpadding_tokens": 2298, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28953659499995865, "nonpadding_tokens": 9257, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28802351800004544, "nonpadding_tokens": 15536, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28555922600003214, "nonpadding_tokens": 6384, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2861431469999616, "nonpadding_tokens": 3327, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000358148000032088, "prefetch": {"data_wait_seconds": 0.01215894700108322, "consumed_batches": 1296, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1846, "tokens": 140609955, "loss_tokens": 79747, "loss": 3.525548374860496, "lr": 0.00029960942753676814, "optimizer_lrs": [0.019973961835784544, 0.00029960942753676814], "gradient_norm": 1.2647241353988647, "seconds": 0.31019900900002995, "nonpadding_tokens_per_second": 257083.34870919044, "ranks": [{"compute_seconds": 0.30720527500000117, "nonpadding_tokens": 4742, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3045572250000532, "nonpadding_tokens": 9762, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3063783910000666, "nonpadding_tokens": 11120, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3054892300000347, "nonpadding_tokens": 1117, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30785961100002623, "nonpadding_tokens": 19101, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30515981999997166, "nonpadding_tokens": 22064, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3074808229999917, "nonpadding_tokens": 6391, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30431119000002127, "nonpadding_tokens": 5450, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040241199997126387, "prefetch": {"data_wait_seconds": 0.012167754001055187, "consumed_batches": 1297, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1847, "tokens": 140671696, "loss_tokens": 61741, "loss": 2.2805345617174972, "lr": 0.0002996090277545558, "optimizer_lrs": [0.019973935183637055, 0.0002996090277545558], "gradient_norm": 0.44200921058654785, "seconds": 0.3587943180000366, "nonpadding_tokens_per_second": 172079.09072850397, "ranks": [{"compute_seconds": 0.27622102099996937, "nonpadding_tokens": 10950, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2813749540000572, "nonpadding_tokens": 4135, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27813029000003553, "nonpadding_tokens": 3060, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27883338099991306, "nonpadding_tokens": 13289, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27711318900003334, "nonpadding_tokens": 1942, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27839459899996655, "nonpadding_tokens": 10530, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2812741280000637, "nonpadding_tokens": 13722, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2774687859999858, "nonpadding_tokens": 4113, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.014618593000022884, "prefetch": {"data_wait_seconds": 0.012175931001095819, "consumed_batches": 1298, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1848, "tokens": 140720821, "loss_tokens": 49125, "loss": 2.464625159033079, "lr": 0.0002996087095169543, "optimizer_lrs": [0.019973913967796953, 0.0002996087095169543], "gradient_norm": 0.45669183135032654, "seconds": 0.29769265900006303, "nonpadding_tokens_per_second": 165019.18510523162, "ranks": [{"compute_seconds": 0.2954556260000345, "nonpadding_tokens": 11695, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29470041299998684, "nonpadding_tokens": 2404, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.294141030999981, "nonpadding_tokens": 6513, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29264330099999825, "nonpadding_tokens": 5706, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.295109884999988, "nonpadding_tokens": 14227, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2923007409999627, "nonpadding_tokens": 4299, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29076842300003136, "nonpadding_tokens": 2483, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29162910399998054, "nonpadding_tokens": 1798, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003576450000082332, "prefetch": {"data_wait_seconds": 0.012184992001110118, "consumed_batches": 1299, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1849, "tokens": 140783107, "loss_tokens": 62286, "loss": 2.308728335821854, "lr": 0.0002996083058349598, "optimizer_lrs": [0.01997388705566399, 0.0002996083058349598], "gradient_norm": 0.48347869515419006, "seconds": 0.2894443200000296, "nonpadding_tokens_per_second": 215191.6472224904, "ranks": [{"compute_seconds": 0.2867211139999881, "nonpadding_tokens": 2291, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2872044050000113, "nonpadding_tokens": 2746, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28413694799996847, "nonpadding_tokens": 8809, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2863779000000477, "nonpadding_tokens": 6616, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2869195790000276, "nonpadding_tokens": 16021, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28576710300001196, "nonpadding_tokens": 8040, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28660197099998186, "nonpadding_tokens": 11916, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28371640400007436, "nonpadding_tokens": 5847, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003421850000222548, "prefetch": {"data_wait_seconds": 0.01219403800116936, "consumed_batches": 1300, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1850, "tokens": 140861036, "loss_tokens": 77929, "loss": 2.534605618575883, "lr": 0.0002996078004763991, "optimizer_lrs": [0.019973853365093275, 0.0002996078004763991], "gradient_norm": 0.6487523317337036, "seconds": 0.2528938460000063, "nonpadding_tokens_per_second": 308149.0563435777, "ranks": [{"compute_seconds": 0.24811927400003242, "nonpadding_tokens": 7564, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2468065939999633, "nonpadding_tokens": 13896, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2492886920000501, "nonpadding_tokens": 17050, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24959467899998344, "nonpadding_tokens": 8721, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24647285000003194, "nonpadding_tokens": 4759, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24860395199993945, "nonpadding_tokens": 8087, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2462195260000044, "nonpadding_tokens": 8817, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2505399679999982, "nonpadding_tokens": 9035, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042648700002700934, "prefetch": {"data_wait_seconds": 0.01220413500114148, "consumed_batches": 1301, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1851, "tokens": 140939759, "loss_tokens": 78723, "loss": 2.2398449309604564, "lr": 0.0002996072896385527, "optimizer_lrs": [0.019973819309236848, 0.0002996072896385527], "gradient_norm": 0.4314545691013336, "seconds": 0.2668295510000007, "nonpadding_tokens_per_second": 295031.04024636233, "ranks": [{"compute_seconds": 0.2229640840000684, "nonpadding_tokens": 2743, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23144228300009217, "nonpadding_tokens": 11450, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22990525499994874, "nonpadding_tokens": 6162, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23079389800000172, "nonpadding_tokens": 14622, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23110605400006534, "nonpadding_tokens": 6344, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2282462509999732, "nonpadding_tokens": 16679, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22671894400002657, "nonpadding_tokens": 16532, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23214523000001464, "nonpadding_tokens": 4191, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.010428719000060482, "prefetch": {"data_wait_seconds": 0.01221289000120862, "consumed_batches": 1302, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1852, "tokens": 141040455, "loss_tokens": 100696, "loss": 2.562851770427822, "lr": 0.0002996066357328042, "optimizer_lrs": [0.01997377571552028, 0.0002996066357328042], "gradient_norm": 2.7579476833343506, "seconds": 0.22990602900006252, "nonpadding_tokens_per_second": 437987.6440733645, "ranks": [{"compute_seconds": 0.2259519490000912, "nonpadding_tokens": 4421, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2277170649999789, "nonpadding_tokens": 25698, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22648459499998808, "nonpadding_tokens": 19496, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22566091700002744, "nonpadding_tokens": 18553, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22382096899991666, "nonpadding_tokens": 7188, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22528178200002458, "nonpadding_tokens": 5721, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22399033699991833, "nonpadding_tokens": 5241, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2243914919999952, "nonpadding_tokens": 14378, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003770690000237664, "prefetch": {"data_wait_seconds": 0.01222213900116742, "consumed_batches": 1303, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1853, "tokens": 141108454, "loss_tokens": 67999, "loss": 2.40436371306931, "lr": 0.0002996061938495599, "optimizer_lrs": [0.019973746256637327, 0.0002996061938495599], "gradient_norm": 0.6184415221214294, "seconds": 0.32057003400007034, "nonpadding_tokens_per_second": 212119.01546600915, "ranks": [{"compute_seconds": 0.3181203710000773, "nonpadding_tokens": 9820, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3183198119999133, "nonpadding_tokens": 8224, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3172197829999277, "nonpadding_tokens": 10760, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31804134899994096, "nonpadding_tokens": 2558, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31425522300003195, "nonpadding_tokens": 3612, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31583952500000123, "nonpadding_tokens": 11921, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31393123499992726, "nonpadding_tokens": 9068, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31468560200005413, "nonpadding_tokens": 12036, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003532710001081796, "prefetch": {"data_wait_seconds": 0.012230724001213389, "consumed_batches": 1304, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1854, "tokens": 141166051, "loss_tokens": 57597, "loss": 2.4753234217928015, "lr": 0.0002996058193686949, "optimizer_lrs": [0.01997372129124633, 0.0002996058193686949], "gradient_norm": 0.41605544090270996, "seconds": 0.2806631290000041, "nonpadding_tokens_per_second": 205217.55103784637, "ranks": [{"compute_seconds": 0.27618514499999947, "nonpadding_tokens": 9974, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2780837120000115, "nonpadding_tokens": 10308, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2752816670000584, "nonpadding_tokens": 5139, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.278431114, "nonpadding_tokens": 3183, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27382889099999375, "nonpadding_tokens": 11169, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.276969544999929, "nonpadding_tokens": 5317, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27348942499997975, "nonpadding_tokens": 8582, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2747253819999287, "nonpadding_tokens": 3925, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037247000000206754, "prefetch": {"data_wait_seconds": 0.012240922001254262, "consumed_batches": 1305, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1855, "tokens": 141212623, "loss_tokens": 46572, "loss": 2.8138571056643475, "lr": 0.0002996055164396028, "optimizer_lrs": [0.019973701095973524, 0.0002996055164396028], "gradient_norm": 0.6110038757324219, "seconds": 0.24275406999993265, "nonpadding_tokens_per_second": 191848.48270520417, "ranks": [{"compute_seconds": 0.24050249900005838, "nonpadding_tokens": 4682, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23574077099999613, "nonpadding_tokens": 14650, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2366035989999773, "nonpadding_tokens": 3753, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23733043999993697, "nonpadding_tokens": 5296, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.239794006000011, "nonpadding_tokens": 4602, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23855536200005645, "nonpadding_tokens": 6081, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23581438099995466, "nonpadding_tokens": 3988, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24031781700000465, "nonpadding_tokens": 3520, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039438700002847327, "prefetch": {"data_wait_seconds": 0.012248646001197017, "consumed_batches": 1306, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1856, "tokens": 141260041, "loss_tokens": 47418, "loss": 2.8473628158083426, "lr": 0.00029960520788830514, "optimizer_lrs": [0.01997368052588701, 0.00029960520788830514], "gradient_norm": 0.45128270983695984, "seconds": 0.30700243399996907, "nonpadding_tokens_per_second": 154454.80148865783, "ranks": [{"compute_seconds": 0.30401373000006515, "nonpadding_tokens": 2819, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30467609699996956, "nonpadding_tokens": 13584, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3037266940000336, "nonpadding_tokens": 2780, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30232636300002014, "nonpadding_tokens": 3410, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30461744500007626, "nonpadding_tokens": 3841, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30166769800007387, "nonpadding_tokens": 10893, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3005658360000325, "nonpadding_tokens": 3252, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30133657000010317, "nonpadding_tokens": 6839, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003681399999777568, "prefetch": {"data_wait_seconds": 0.012256355001184716, "consumed_batches": 1307, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1857, "tokens": 141340707, "loss_tokens": 80666, "loss": 2.748641001165299, "lr": 0.00029960468271387727, "optimizer_lrs": [0.019973645514258485, 0.00029960468271387727], "gradient_norm": 0.3786632716655731, "seconds": 0.5057473429999391, "nonpadding_tokens_per_second": 159498.61352016972, "ranks": [{"compute_seconds": 0.5032640999999103, "nonpadding_tokens": 11050, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4991582220000055, "nonpadding_tokens": 11388, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.5018767049999724, "nonpadding_tokens": 10504, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.502559228999985, "nonpadding_tokens": 6814, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4981128590000026, "nonpadding_tokens": 4394, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5004426039999998, "nonpadding_tokens": 8864, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.49837711600002876, "nonpadding_tokens": 11190, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.49868739200007894, "nonpadding_tokens": 16462, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038929899994855077, "prefetch": {"data_wait_seconds": 0.012265712001251927, "consumed_batches": 1308, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1858, "tokens": 141390879, "loss_tokens": 50172, "loss": 2.4313002334469425, "lr": 0.0002996043558942254, "optimizer_lrs": [0.019973623726281694, 0.0002996043558942254], "gradient_norm": 0.6280559301376343, "seconds": 0.22682994299998427, "nonpadding_tokens_per_second": 221187.72917032157, "ranks": [{"compute_seconds": 0.22264831500001492, "nonpadding_tokens": 6069, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2244219010000279, "nonpadding_tokens": 6188, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2218707639999593, "nonpadding_tokens": 3094, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22219808199997715, "nonpadding_tokens": 5110, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2226197710000406, "nonpadding_tokens": 4386, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22016176900001483, "nonpadding_tokens": 4248, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2186021489999348, "nonpadding_tokens": 14399, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21944659100006447, "nonpadding_tokens": 6678, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038300199992136186, "prefetch": {"data_wait_seconds": 0.012275001001356145, "consumed_batches": 1309, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1859, "tokens": 141450410, "loss_tokens": 59531, "loss": 2.4509643610051906, "lr": 0.0002996039679352716, "optimizer_lrs": [0.019973597862351443, 0.0002996039679352716], "gradient_norm": 0.3953641355037689, "seconds": 0.2515282260000049, "nonpadding_tokens_per_second": 236677.21490628587, "ranks": [{"compute_seconds": 0.24810915000000477, "nonpadding_tokens": 5792, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24500580000005812, "nonpadding_tokens": 13388, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24486076900006992, "nonpadding_tokens": 7901, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2457870630000798, "nonpadding_tokens": 7103, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24358995300008246, "nonpadding_tokens": 8392, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24627999699998782, "nonpadding_tokens": 1895, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24837270499995157, "nonpadding_tokens": 12235, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2491623459999346, "nonpadding_tokens": 2825, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035689200001343124, "prefetch": {"data_wait_seconds": 0.012284676001357298, "consumed_batches": 1310, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1860, "tokens": 141528359, "loss_tokens": 77949, "loss": 2.2999545375822654, "lr": 0.00029960345966060234, "optimizer_lrs": [0.01997356397737349, 0.00029960345966060234], "gradient_norm": 0.465632826089859, "seconds": 0.28410629500001505, "nonpadding_tokens_per_second": 274365.6207969481, "ranks": [{"compute_seconds": 0.21815964400002485, "nonpadding_tokens": 13638, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22588173899998765, "nonpadding_tokens": 5941, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22799800299992512, "nonpadding_tokens": 14587, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22607016000006297, "nonpadding_tokens": 6076, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2292312710000033, "nonpadding_tokens": 5663, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2264988209999501, "nonpadding_tokens": 3767, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22871130999999423, "nonpadding_tokens": 18084, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22950447499999882, "nonpadding_tokens": 10193, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.03263682800002243, "prefetch": {"data_wait_seconds": 0.012293082001363587, "consumed_batches": 1311, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1861, "tokens": 141624594, "loss_tokens": 96235, "loss": 2.0147617161116016, "lr": 0.00029960283170118204, "optimizer_lrs": [0.019973522113412138, 0.00029960283170118204], "gradient_norm": 1.3646438121795654, "seconds": 0.326170798000021, "nonpadding_tokens_per_second": 295044.8065555942, "ranks": [{"compute_seconds": 0.32292685999993864, "nonpadding_tokens": 9556, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3193186790000482, "nonpadding_tokens": 4812, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3200756769999771, "nonpadding_tokens": 4307, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32265409399997225, "nonpadding_tokens": 5476, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3230833350000921, "nonpadding_tokens": 10008, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.322141184999964, "nonpadding_tokens": 20098, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32277923800006647, "nonpadding_tokens": 23511, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3236807499999941, "nonpadding_tokens": 18467, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047414499999831605, "prefetch": {"data_wait_seconds": 0.012302889001446715, "consumed_batches": 1312, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1862, "tokens": 141757514, "loss_tokens": 132920, "loss": 2.2102002144146855, "lr": 0.00029960196354640643, "optimizer_lrs": [0.019973464236427096, 0.00029960196354640643], "gradient_norm": 2.2353579998016357, "seconds": 0.3178436179999835, "nonpadding_tokens_per_second": 418193.07506122993, "ranks": [{"compute_seconds": 0.31557934500006013, "nonpadding_tokens": 18035, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31518946300002426, "nonpadding_tokens": 15391, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31176806400003443, "nonpadding_tokens": 19965, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3123645719999786, "nonpadding_tokens": 11830, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31105031300000974, "nonpadding_tokens": 15687, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31224856300002557, "nonpadding_tokens": 17661, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3145320329999777, "nonpadding_tokens": 20325, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3115742830000272, "nonpadding_tokens": 14026, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003895129999591518, "prefetch": {"data_wait_seconds": 0.012325356001340992, "consumed_batches": 1313, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1863, "tokens": 141865678, "loss_tokens": 108164, "loss": 1.9145753208091416, "lr": 0.00029960125638475645, "optimizer_lrs": [0.0199734170923171, 0.00029960125638475645], "gradient_norm": 0.5978091955184937, "seconds": 0.31985092699994766, "nonpadding_tokens_per_second": 338170.0375688381, "ranks": [{"compute_seconds": 0.3159607469999628, "nonpadding_tokens": 23034, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3172278100000767, "nonpadding_tokens": 21923, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3143456719999449, "nonpadding_tokens": 24999, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31664715200008686, "nonpadding_tokens": 15718, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31755279899994093, "nonpadding_tokens": 9952, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.316267559000039, "nonpadding_tokens": 6191, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31709976299998743, "nonpadding_tokens": 1255, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3137828419999096, "nonpadding_tokens": 5092, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034837999999126623, "prefetch": {"data_wait_seconds": 0.012333772001355214, "consumed_batches": 1314, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1864, "tokens": 141924864, "loss_tokens": 59186, "loss": 2.3705822534045216, "lr": 0.00029960086916938714, "optimizer_lrs": [0.019973391277959145, 0.00029960086916938714], "gradient_norm": 0.47630181908607483, "seconds": 0.27597629700005655, "nonpadding_tokens_per_second": 214460.44694189035, "ranks": [{"compute_seconds": 0.2736357899999575, "nonpadding_tokens": 3416, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2733087799999794, "nonpadding_tokens": 11557, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27254778399992574, "nonpadding_tokens": 10972, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27254751600003146, "nonpadding_tokens": 8389, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2692175380000208, "nonpadding_tokens": 3939, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2704574950000733, "nonpadding_tokens": 4288, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2694236730000057, "nonpadding_tokens": 8052, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2695412399999668, "nonpadding_tokens": 8573, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034694699991177913, "prefetch": {"data_wait_seconds": 0.01234229800134301, "consumed_batches": 1315, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1865, "tokens": 141987062, "loss_tokens": 62198, "loss": 2.344240871892987, "lr": 0.0002996004620462604, "optimizer_lrs": [0.01997336413641736, 0.0002996004620462604], "gradient_norm": 0.7771055698394775, "seconds": 0.33965165799997976, "nonpadding_tokens_per_second": 183122.91000211667, "ranks": [{"compute_seconds": 0.3349489319999748, "nonpadding_tokens": 24610, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33688504099995953, "nonpadding_tokens": 5632, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.334180554999989, "nonpadding_tokens": 5245, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33468146600000637, "nonpadding_tokens": 3813, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3370807329999934, "nonpadding_tokens": 5888, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33599244599997746, "nonpadding_tokens": 1520, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33313304800003607, "nonpadding_tokens": 3238, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33344782800008943, "nonpadding_tokens": 12252, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003746380000393401, "prefetch": {"data_wait_seconds": 0.01235104800134934, "consumed_batches": 1316, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1866, "tokens": 142040267, "loss_tokens": 53205, "loss": 2.3763313774551262, "lr": 0.00029960011362325216, "optimizer_lrs": [0.019973340908216813, 0.00029960011362325216], "gradient_norm": 0.5254471302032471, "seconds": 0.288657882999928, "nonpadding_tokens_per_second": 184318.54154495156, "ranks": [{"compute_seconds": 0.2857661430000462, "nonpadding_tokens": 3922, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2857029279999779, "nonpadding_tokens": 2645, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2850600819999727, "nonpadding_tokens": 5072, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2852641340000446, "nonpadding_tokens": 10108, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2820412020000731, "nonpadding_tokens": 4733, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28447979400004897, "nonpadding_tokens": 9026, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28145462700001644, "nonpadding_tokens": 5731, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2861443759999247, "nonpadding_tokens": 11968, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037234499995975057, "prefetch": {"data_wait_seconds": 0.012360753001374292, "consumed_batches": 1317, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1867, "tokens": 142097115, "loss_tokens": 56848, "loss": 2.6250918018224034, "lr": 0.000299599741175808, "optimizer_lrs": [0.0199733160783872, 0.000299599741175808], "gradient_norm": 0.502148449420929, "seconds": 0.2247847759999786, "nonpadding_tokens_per_second": 252899.68925655985, "ranks": [{"compute_seconds": 0.22248565600000347, "nonpadding_tokens": 1603, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21868298200001846, "nonpadding_tokens": 11805, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22154650899994976, "nonpadding_tokens": 3680, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21982373399998778, "nonpadding_tokens": 4310, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21837488799997118, "nonpadding_tokens": 7572, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22108708899997964, "nonpadding_tokens": 2687, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2221949870000799, "nonpadding_tokens": 12011, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21914945099990746, "nonpadding_tokens": 13180, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003945789999306726, "prefetch": {"data_wait_seconds": 0.01236992500135159, "consumed_batches": 1318, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1868, "tokens": 142160431, "loss_tokens": 63316, "loss": 2.224813583849264, "lr": 0.0002995993261486062, "optimizer_lrs": [0.019973288409907083, 0.0002995993261486062], "gradient_norm": 0.37947168946266174, "seconds": 0.30914446899998893, "nonpadding_tokens_per_second": 204810.39238648734, "ranks": [{"compute_seconds": 0.30649161799999547, "nonpadding_tokens": 13774, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3029912819999936, "nonpadding_tokens": 7828, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30536984799994116, "nonpadding_tokens": 2905, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3039031079999859, "nonpadding_tokens": 10219, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30219041899999866, "nonpadding_tokens": 4638, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3037108209999815, "nonpadding_tokens": 13530, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3025348659999736, "nonpadding_tokens": 6369, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3068083809999962, "nonpadding_tokens": 4053, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003880680000065695, "prefetch": {"data_wait_seconds": 0.012379106001347395, "consumed_batches": 1319, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1869, "tokens": 142221402, "loss_tokens": 60971, "loss": 2.20125372103131, "lr": 0.00029959892628958305, "optimizer_lrs": [0.01997326175263887, 0.00029959892628958305], "gradient_norm": 0.4347480535507202, "seconds": 0.23173626800007696, "nonpadding_tokens_per_second": 263105.1260391392, "ranks": [{"compute_seconds": 0.22909368900002391, "nonpadding_tokens": 2621, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2290606060000755, "nonpadding_tokens": 9245, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22588119699992149, "nonpadding_tokens": 13103, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22818860899997162, "nonpadding_tokens": 20916, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22890599900006237, "nonpadding_tokens": 3276, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22577496299993527, "nonpadding_tokens": 5815, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22459014300000035, "nonpadding_tokens": 3871, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.228778792000071, "nonpadding_tokens": 2124, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039727599994421325, "prefetch": {"data_wait_seconds": 0.012388662001356465, "consumed_batches": 1320, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1870, "tokens": 142285847, "loss_tokens": 64445, "loss": 2.4460901737916054, "lr": 0.0002995985034309608, "optimizer_lrs": [0.019973233562064055, 0.0002995985034309608], "gradient_norm": 0.39868927001953125, "seconds": 0.3740560539999933, "nonpadding_tokens_per_second": 172287.01236312874, "ranks": [{"compute_seconds": 0.36977011800001947, "nonpadding_tokens": 8904, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3679861020000317, "nonpadding_tokens": 4098, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3703939250000303, "nonpadding_tokens": 2766, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3689532070000041, "nonpadding_tokens": 12654, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3714751500000375, "nonpadding_tokens": 8306, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3703920419999349, "nonpadding_tokens": 9732, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3711568079999097, "nonpadding_tokens": 10476, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3681962409999642, "nonpadding_tokens": 7509, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004176810000444675, "prefetch": {"data_wait_seconds": 0.012397232001333691, "consumed_batches": 1321, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1871, "tokens": 142318654, "loss_tokens": 32807, "loss": 2.511397148474411, "lr": 0.00029959828808097714, "optimizer_lrs": [0.019973219205398476, 0.00029959828808097714], "gradient_norm": 0.48032984137535095, "seconds": 0.2188278349999564, "nonpadding_tokens_per_second": 149921.51249865693, "ranks": [{"compute_seconds": 0.21542922099990847, "nonpadding_tokens": 1529, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21547677099999873, "nonpadding_tokens": 4134, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21511805699992692, "nonpadding_tokens": 9490, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21284683199996834, "nonpadding_tokens": 1879, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21568898300006367, "nonpadding_tokens": 4566, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21323180499996397, "nonpadding_tokens": 3047, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2113528029999543, "nonpadding_tokens": 5244, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21649399900002209, "nonpadding_tokens": 2918, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003902959999777522, "prefetch": {"data_wait_seconds": 0.012406410001361223, "consumed_batches": 1322, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1872, "tokens": 142379405, "loss_tokens": 60751, "loss": 2.5210001687215025, "lr": 0.0002995978891502338, "optimizer_lrs": [0.01997319261001559, 0.0002995978891502338], "gradient_norm": 0.39004185795783997, "seconds": 0.2884618220000448, "nonpadding_tokens_per_second": 210603.25965766996, "ranks": [{"compute_seconds": 0.2859402569999929, "nonpadding_tokens": 11174, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2862466649999078, "nonpadding_tokens": 6501, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28353133699999944, "nonpadding_tokens": 12467, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28415136500007065, "nonpadding_tokens": 3557, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2824972300001036, "nonpadding_tokens": 6954, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28528667000000496, "nonpadding_tokens": 3669, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28618787699997483, "nonpadding_tokens": 3616, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28320734999999786, "nonpadding_tokens": 12813, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035957000000053085, "prefetch": {"data_wait_seconds": 0.01241440400129079, "consumed_batches": 1323, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1873, "tokens": 142423475, "loss_tokens": 44070, "loss": 2.3622293013955074, "lr": 0.00029959759963411094, "optimizer_lrs": [0.01997317330894073, 0.00029959759963411094], "gradient_norm": 0.5187169313430786, "seconds": 0.23951654099994357, "nonpadding_tokens_per_second": 183995.64312349676, "ranks": [{"compute_seconds": 0.2363129550000167, "nonpadding_tokens": 2934, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23699345500006075, "nonpadding_tokens": 4785, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23612964100004774, "nonpadding_tokens": 3948, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23454322100008085, "nonpadding_tokens": 3121, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23277808399996047, "nonpadding_tokens": 11057, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23561332400004176, "nonpadding_tokens": 9321, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23279967999997098, "nonpadding_tokens": 4071, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23725639400004184, "nonpadding_tokens": 4833, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036477200001172605, "prefetch": {"data_wait_seconds": 0.012423066001204575, "consumed_batches": 1324, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1874, "tokens": 142498943, "loss_tokens": 75468, "loss": 2.6140226652355967, "lr": 0.00029959710360845707, "optimizer_lrs": [0.019973140240563805, 0.00029959710360845707], "gradient_norm": 0.44070273637771606, "seconds": 0.2868138490000547, "nonpadding_tokens_per_second": 263125.3695144463, "ranks": [{"compute_seconds": 0.2845636189999823, "nonpadding_tokens": 6287, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28087403500001074, "nonpadding_tokens": 3423, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28373453699998663, "nonpadding_tokens": 12321, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2821577739999839, "nonpadding_tokens": 12679, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2807427950000374, "nonpadding_tokens": 18263, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28365609100001166, "nonpadding_tokens": 13055, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2806813870000724, "nonpadding_tokens": 1620, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28126693000001524, "nonpadding_tokens": 7820, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033955000003516034, "prefetch": {"data_wait_seconds": 0.012431681001160655, "consumed_batches": 1325, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1875, "tokens": 142560454, "loss_tokens": 61511, "loss": 2.4005358289574223, "lr": 0.0002995966990917868, "optimizer_lrs": [0.019973113272785788, 0.0002995966990917868], "gradient_norm": 0.3907665014266968, "seconds": 0.32476179399998273, "nonpadding_tokens_per_second": 189403.43703115298, "ranks": [{"compute_seconds": 0.31985039599999254, "nonpadding_tokens": 4737, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3222526329999482, "nonpadding_tokens": 5984, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31911590499998965, "nonpadding_tokens": 5433, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3215839460000325, "nonpadding_tokens": 5691, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3180971760000375, "nonpadding_tokens": 21741, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3192074840000032, "nonpadding_tokens": 1476, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31803102200001376, "nonpadding_tokens": 12860, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3224256760000799, "nonpadding_tokens": 3589, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038806299994575966, "prefetch": {"data_wait_seconds": 0.01244027600110087, "consumed_batches": 1326, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1876, "tokens": 142623956, "loss_tokens": 63502, "loss": 2.7735398589808193, "lr": 0.00029959628126903027, "optimizer_lrs": [0.01997308541793535, 0.00029959628126903027], "gradient_norm": 0.6562377214431763, "seconds": 0.3138765029999604, "nonpadding_tokens_per_second": 202315.23988913567, "ranks": [{"compute_seconds": 0.3112164679999978, "nonpadding_tokens": 12099, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31151028100009626, "nonpadding_tokens": 2577, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30853606300001957, "nonpadding_tokens": 5789, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3112515509999412, "nonpadding_tokens": 11348, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30745235099993806, "nonpadding_tokens": 4393, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30914328000005753, "nonpadding_tokens": 17705, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31153782599994884, "nonpadding_tokens": 7601, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30776169499995376, "nonpadding_tokens": 1990, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003668660000357704, "prefetch": {"data_wait_seconds": 0.012449196001128371, "consumed_batches": 1327, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1877, "tokens": 142671910, "loss_tokens": 47954, "loss": 3.0039318278975684, "lr": 0.00029959596560397427, "optimizer_lrs": [0.019973064373598286, 0.00029959596560397427], "gradient_norm": 0.7954875826835632, "seconds": 0.2868777379999301, "nonpadding_tokens_per_second": 167158.31745721475, "ranks": [{"compute_seconds": 0.28458679699997447, "nonpadding_tokens": 1761, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28064985499997874, "nonpadding_tokens": 2392, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2812772910000376, "nonpadding_tokens": 2003, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2839635149999822, "nonpadding_tokens": 3076, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2802206669999805, "nonpadding_tokens": 13851, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2834065639999608, "nonpadding_tokens": 4071, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2840949990001036, "nonpadding_tokens": 4072, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2811180969999896, "nonpadding_tokens": 16728, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003542670000342696, "prefetch": {"data_wait_seconds": 0.012458963001179768, "consumed_batches": 1328, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1878, "tokens": 142736769, "loss_tokens": 64859, "loss": 2.640023213817666, "lr": 0.0002995955384630259, "optimizer_lrs": [0.01997303589753506, 0.0002995955384630259], "gradient_norm": 0.5264529585838318, "seconds": 0.36374761000001854, "nonpadding_tokens_per_second": 178307.70077086333, "ranks": [{"compute_seconds": 0.3604851980001058, "nonpadding_tokens": 12965, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3574726289999717, "nonpadding_tokens": 7591, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36025265899991155, "nonpadding_tokens": 4987, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3609563050000588, "nonpadding_tokens": 6359, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3570677259999684, "nonpadding_tokens": 4865, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35835081200002605, "nonpadding_tokens": 15926, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3575256059999674, "nonpadding_tokens": 4845, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3614567030000444, "nonpadding_tokens": 7321, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003562100000635837, "prefetch": {"data_wait_seconds": 0.0124681150011412, "consumed_batches": 1329, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1879, "tokens": 142807415, "loss_tokens": 70646, "loss": 2.661516796775472, "lr": 0.0002995950729543377, "optimizer_lrs": [0.019973004863622516, 0.0002995950729543377], "gradient_norm": 0.42993974685668945, "seconds": 0.31054943500009813, "nonpadding_tokens_per_second": 227487.13099406436, "ranks": [{"compute_seconds": 0.3082515059999196, "nonpadding_tokens": 4527, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30822214900001654, "nonpadding_tokens": 11306, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30740305799997714, "nonpadding_tokens": 10214, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30599914599997646, "nonpadding_tokens": 3533, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30400463099999797, "nonpadding_tokens": 4839, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.305526642000018, "nonpadding_tokens": 18843, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3039159769999742, "nonpadding_tokens": 12812, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3048397599999362, "nonpadding_tokens": 4572, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003629670000009355, "prefetch": {"data_wait_seconds": 0.012476854001150173, "consumed_batches": 1330, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1880, "tokens": 142900727, "loss_tokens": 93312, "loss": 2.1963453253600824, "lr": 0.00029959445768251307, "optimizer_lrs": [0.019972963845500874, 0.00029959445768251307], "gradient_norm": 0.7685824036598206, "seconds": 0.2716931070000328, "nonpadding_tokens_per_second": 343446.3282132098, "ranks": [{"compute_seconds": 0.22726300099998298, "nonpadding_tokens": 9243, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22709836599995015, "nonpadding_tokens": 3038, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2276715079999576, "nonpadding_tokens": 13421, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22838750999994772, "nonpadding_tokens": 20395, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22648064600002726, "nonpadding_tokens": 13228, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2277715459999854, "nonpadding_tokens": 20894, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22674086899996837, "nonpadding_tokens": 11227, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23111080000001039, "nonpadding_tokens": 1866, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.027438826999969024, "prefetch": {"data_wait_seconds": 0.012485560001053386, "consumed_batches": 1331, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1881, "tokens": 142953332, "loss_tokens": 52605, "loss": 2.6570620663435034, "lr": 0.00029959411061507264, "optimizer_lrs": [0.01997294070767151, 0.00029959411061507264], "gradient_norm": 0.4088943600654602, "seconds": 0.25455177599997114, "nonpadding_tokens_per_second": 206657.36781190624, "ranks": [{"compute_seconds": 0.2505871759999536, "nonpadding_tokens": 3770, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25151835899998787, "nonpadding_tokens": 3194, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25088350800001535, "nonpadding_tokens": 5655, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24922362500001327, "nonpadding_tokens": 1957, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2517898679999462, "nonpadding_tokens": 8087, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2485344400000713, "nonpadding_tokens": 17163, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25119703600000776, "nonpadding_tokens": 7068, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2521853869999404, "nonpadding_tokens": 5711, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000378125999986878, "prefetch": {"data_wait_seconds": 0.01249537700103076, "consumed_batches": 1332, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1882, "tokens": 143054234, "loss_tokens": 100902, "loss": 2.2466441757844247, "lr": 0.00029959344448788445, "optimizer_lrs": [0.019972896299192296, 0.00029959344448788445], "gradient_norm": 0.4929957389831543, "seconds": 0.26230564399998, "nonpadding_tokens_per_second": 384673.3850683277, "ranks": [{"compute_seconds": 0.25963422899997113, "nonpadding_tokens": 8025, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25943461899998965, "nonpadding_tokens": 6080, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2564704939999274, "nonpadding_tokens": 8906, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25908751200006463, "nonpadding_tokens": 26436, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.259466512999893, "nonpadding_tokens": 25939, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2584711839999727, "nonpadding_tokens": 10866, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.259287449999988, "nonpadding_tokens": 1545, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2599886540000398, "nonpadding_tokens": 13105, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038024499997391104, "prefetch": {"data_wait_seconds": 0.01250534800097114, "consumed_batches": 1333, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1883, "tokens": 143133476, "loss_tokens": 79242, "loss": 2.307696873185937, "lr": 0.0002995929209717399, "optimizer_lrs": [0.019972861398115993, 0.0002995929209717399], "gradient_norm": 0.36793142557144165, "seconds": 0.3634859300000244, "nonpadding_tokens_per_second": 218005.6873177861, "ranks": [{"compute_seconds": 0.3016571409999642, "nonpadding_tokens": 11255, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2981515900000886, "nonpadding_tokens": 7642, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3004933769999525, "nonpadding_tokens": 6404, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2956320240000423, "nonpadding_tokens": 10542, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30156441600001926, "nonpadding_tokens": 18791, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2967794669999648, "nonpadding_tokens": 8969, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3012611740000466, "nonpadding_tokens": 10863, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2947278030000007, "nonpadding_tokens": 4776, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.03247670299992933, "prefetch": {"data_wait_seconds": 0.01251387800103032, "consumed_batches": 1334, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1884, "tokens": 143178581, "loss_tokens": 45105, "loss": 2.644140928112183, "lr": 0.0002995926228331455, "optimizer_lrs": [0.019972841522209704, 0.0002995926228331455], "gradient_norm": 0.5027112364768982, "seconds": 0.22545769900000323, "nonpadding_tokens_per_second": 200059.70166492008, "ranks": [{"compute_seconds": 0.2226763400000209, "nonpadding_tokens": 2903, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2227866629999653, "nonpadding_tokens": 9191, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21995916900004886, "nonpadding_tokens": 6914, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22047132900001998, "nonpadding_tokens": 7601, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21885998599998402, "nonpadding_tokens": 3486, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21982617900005152, "nonpadding_tokens": 5663, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.218727496999918, "nonpadding_tokens": 3315, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22305364900000768, "nonpadding_tokens": 6032, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004052670000191938, "prefetch": {"data_wait_seconds": 0.012524277000991901, "consumed_batches": 1335, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1885, "tokens": 143238900, "loss_tokens": 60319, "loss": 2.132830373721381, "lr": 0.00029959222396154695, "optimizer_lrs": [0.0199728149307698, 0.00029959222396154695], "gradient_norm": 0.6470826268196106, "seconds": 0.3423183429999881, "nonpadding_tokens_per_second": 176207.3264067012, "ranks": [{"compute_seconds": 0.3392128440000306, "nonpadding_tokens": 3162, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3358526319999555, "nonpadding_tokens": 4384, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3387632310000299, "nonpadding_tokens": 11076, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33739838199994665, "nonpadding_tokens": 3592, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3355262079999193, "nonpadding_tokens": 10094, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3368914750000158, "nonpadding_tokens": 12350, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3356420590000653, "nonpadding_tokens": 3041, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33998119400007454, "nonpadding_tokens": 12620, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036195200004840444, "prefetch": {"data_wait_seconds": 0.012532687000955889, "consumed_batches": 1336, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1886, "tokens": 143280246, "loss_tokens": 41346, "loss": 2.2233491450200744, "lr": 0.0002995919504401783, "optimizer_lrs": [0.019972796696011888, 0.0002995919504401783], "gradient_norm": 0.4040880501270294, "seconds": 0.2820827880000252, "nonpadding_tokens_per_second": 146573.9908951705, "ranks": [{"compute_seconds": 0.2798089630000504, "nonpadding_tokens": 7745, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2757089319999295, "nonpadding_tokens": 2465, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2783842190000314, "nonpadding_tokens": 3592, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27858073799995964, "nonpadding_tokens": 3567, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2752726740000071, "nonpadding_tokens": 6441, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27769747699994696, "nonpadding_tokens": 8398, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27518171899998833, "nonpadding_tokens": 5653, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27951442099993073, "nonpadding_tokens": 3485, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003692060000730635, "prefetch": {"data_wait_seconds": 0.01254167400095696, "consumed_batches": 1337, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1887, "tokens": 143342587, "loss_tokens": 62341, "loss": 2.8199537122439486, "lr": 0.0002995915378548461, "optimizer_lrs": [0.019972769190323075, 0.0002995915378548461], "gradient_norm": 0.5379387736320496, "seconds": 0.2976747750000186, "nonpadding_tokens_per_second": 209426.54613578226, "ranks": [{"compute_seconds": 0.29289633700000195, "nonpadding_tokens": 10322, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29509910900003433, "nonpadding_tokens": 3145, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2942181640000854, "nonpadding_tokens": 3923, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29457133599998997, "nonpadding_tokens": 2505, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29085721000001286, "nonpadding_tokens": 10645, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2924995690000287, "nonpadding_tokens": 13504, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2948941089999835, "nonpadding_tokens": 10233, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2953253370000084, "nonpadding_tokens": 8064, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035432900006071577, "prefetch": {"data_wait_seconds": 0.012550259001002928, "consumed_batches": 1338, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1888, "tokens": 143393825, "loss_tokens": 51238, "loss": 2.713324339357508, "lr": 0.00029959119859555654, "optimizer_lrs": [0.019972746573037103, 0.00029959119859555654], "gradient_norm": 0.4522855877876282, "seconds": 0.3133419129999311, "nonpadding_tokens_per_second": 163521.05439533482, "ranks": [{"compute_seconds": 0.3087670339999704, "nonpadding_tokens": 11341, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3072892980000006, "nonpadding_tokens": 11146, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3099765460000299, "nonpadding_tokens": 8629, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31099032199995236, "nonpadding_tokens": 2147, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3065937620000341, "nonpadding_tokens": 1725, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.309357322999972, "nonpadding_tokens": 8836, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3108232859999589, "nonpadding_tokens": 3396, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30733755599999313, "nonpadding_tokens": 4018, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035463700010041066, "prefetch": {"data_wait_seconds": 0.01255799900100385, "consumed_batches": 1339, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1889, "tokens": 143486225, "loss_tokens": 92400, "loss": 2.680445752164502, "lr": 0.00029959058643718784, "optimizer_lrs": [0.01997270576247919, 0.00029959058643718784], "gradient_norm": 0.4088227152824402, "seconds": 0.3781471080000074, "nonpadding_tokens_per_second": 244349.3498831629, "ranks": [{"compute_seconds": 0.375055431000078, "nonpadding_tokens": 12168, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37535758100000294, "nonpadding_tokens": 25425, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37502174799999466, "nonpadding_tokens": 6362, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.37556779800002005, "nonpadding_tokens": 9279, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.37129546000005575, "nonpadding_tokens": 8909, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37357415500002844, "nonpadding_tokens": 12522, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3758212129999947, "nonpadding_tokens": 13580, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37214879300006487, "nonpadding_tokens": 4155, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004100100001096507, "prefetch": {"data_wait_seconds": 0.012567258000970583, "consumed_batches": 1340, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1890, "tokens": 143557511, "loss_tokens": 71286, "loss": 2.4001114349241086, "lr": 0.00029959011384848746, "optimizer_lrs": [0.019972674256565834, 0.00029959011384848746], "gradient_norm": 0.39372652769088745, "seconds": 0.35217154400004347, "nonpadding_tokens_per_second": 202418.39868808707, "ranks": [{"compute_seconds": 0.34903591700003744, "nonpadding_tokens": 24565, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34529434299997774, "nonpadding_tokens": 7304, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3462818179999658, "nonpadding_tokens": 3801, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34682287299995096, "nonpadding_tokens": 5744, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.345220591000043, "nonpadding_tokens": 6215, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3463263090000055, "nonpadding_tokens": 3024, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34500234300003285, "nonpadding_tokens": 11995, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34985689299992373, "nonpadding_tokens": 8638, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037581300000510964, "prefetch": {"data_wait_seconds": 0.012576459000868567, "consumed_batches": 1341, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1891, "tokens": 143618592, "loss_tokens": 61081, "loss": 2.232757015274799, "lr": 0.0002995897086970456, "optimizer_lrs": [0.01997264724646971, 0.0002995897086970456], "gradient_norm": 0.9874795079231262, "seconds": 0.2982334850000825, "nonpadding_tokens_per_second": 204809.32917369457, "ranks": [{"compute_seconds": 0.2943165490000865, "nonpadding_tokens": 11441, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29135842599998796, "nonpadding_tokens": 9633, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29148514300004535, "nonpadding_tokens": 6449, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2919382960000121, "nonpadding_tokens": 8531, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2957831869999836, "nonpadding_tokens": 2434, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2933975419999797, "nonpadding_tokens": 7443, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29529447300001266, "nonpadding_tokens": 9490, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2913333530000273, "nonpadding_tokens": 5660, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043325999990884156, "prefetch": {"data_wait_seconds": 0.01258607200095696, "consumed_batches": 1342, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1892, "tokens": 143683725, "loss_tokens": 65133, "loss": 2.452145034007339, "lr": 0.0002995892764484537, "optimizer_lrs": [0.019972618429896914, 0.0002995892764484537], "gradient_norm": 0.5065820813179016, "seconds": 0.3533522879999964, "nonpadding_tokens_per_second": 184328.79087512987, "ranks": [{"compute_seconds": 0.3504481160000523, "nonpadding_tokens": 8663, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3507934630000591, "nonpadding_tokens": 10884, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3478321819999337, "nonpadding_tokens": 3410, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3500415850000991, "nonpadding_tokens": 6044, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34681060299999444, "nonpadding_tokens": 20442, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.349104180999916, "nonpadding_tokens": 6937, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34673018299997693, "nonpadding_tokens": 3068, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35105693899993184, "nonpadding_tokens": 5685, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003494249999675958, "prefetch": {"data_wait_seconds": 0.012594386001026692, "consumed_batches": 1343, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1893, "tokens": 143774212, "loss_tokens": 90487, "loss": 2.426030811055732, "lr": 0.0002995886755634811, "optimizer_lrs": [0.019972578370898744, 0.0002995886755634811], "gradient_norm": 0.4153726398944855, "seconds": 0.28133721999995487, "nonpadding_tokens_per_second": 321631.8125273809, "ranks": [{"compute_seconds": 0.2788231319999568, "nonpadding_tokens": 10359, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2743188409999675, "nonpadding_tokens": 7488, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2752952060000098, "nonpadding_tokens": 9981, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2760558130000845, "nonpadding_tokens": 17411, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2744028890000436, "nonpadding_tokens": 10067, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27578888200002893, "nonpadding_tokens": 15938, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2780046330000232, "nonpadding_tokens": 7087, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2789630050000369, "nonpadding_tokens": 12156, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038285699997686606, "prefetch": {"data_wait_seconds": 0.012602883001093801, "consumed_batches": 1344, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1894, "tokens": 143826594, "loss_tokens": 52382, "loss": 2.5793799038791954, "lr": 0.0002995883275168889, "optimizer_lrs": [0.019972555167792595, 0.0002995883275168889], "gradient_norm": 0.46965837478637695, "seconds": 0.26917160300001797, "nonpadding_tokens_per_second": 194604.48062196406, "ranks": [{"compute_seconds": 0.26585301500006153, "nonpadding_tokens": 2134, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26308502400002, "nonpadding_tokens": 13359, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2635333409999703, "nonpadding_tokens": 2740, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2661065290000124, "nonpadding_tokens": 3748, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26686825600006614, "nonpadding_tokens": 2505, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26389636600003996, "nonpadding_tokens": 9149, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2662989780000089, "nonpadding_tokens": 3913, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2635937710000462, "nonpadding_tokens": 14834, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038408599993999815, "prefetch": {"data_wait_seconds": 0.01261131700114504, "consumed_batches": 1345, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1895, "tokens": 143889462, "loss_tokens": 62868, "loss": 2.6256434612998665, "lr": 0.0002995879096031531, "optimizer_lrs": [0.019972527306876875, 0.0002995879096031531], "gradient_norm": 0.44368064403533936, "seconds": 0.3313147400000389, "nonpadding_tokens_per_second": 189753.10304634384, "ranks": [{"compute_seconds": 0.32618461200002, "nonpadding_tokens": 9215, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32478539300007014, "nonpadding_tokens": 14956, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32767575599996235, "nonpadding_tokens": 10140, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32827356799998597, "nonpadding_tokens": 14718, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3289558689999694, "nonpadding_tokens": 3870, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3266354190000129, "nonpadding_tokens": 2945, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3286267720000069, "nonpadding_tokens": 5904, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3245177930000409, "nonpadding_tokens": 1120, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039544399999158486, "prefetch": {"data_wait_seconds": 0.012619717001101094, "consumed_batches": 1346, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1896, "tokens": 143948134, "loss_tokens": 58672, "loss": 2.506233009357104, "lr": 0.00029958751939127, "optimizer_lrs": [0.019972501292751334, 0.00029958751939127], "gradient_norm": 0.4332686960697174, "seconds": 0.2932055379999383, "nonpadding_tokens_per_second": 200105.36090219533, "ranks": [{"compute_seconds": 0.29078877300003114, "nonpadding_tokens": 12034, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2869663510000464, "nonpadding_tokens": 10407, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28940963000002284, "nonpadding_tokens": 3531, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29028840000000855, "nonpadding_tokens": 6374, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29051979700000174, "nonpadding_tokens": 5064, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28814545600005204, "nonpadding_tokens": 13767, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28577543099993363, "nonpadding_tokens": 1939, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2907741880000003, "nonpadding_tokens": 5556, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003816079999978683, "prefetch": {"data_wait_seconds": 0.012628492001070413, "consumed_batches": 1347, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1897, "tokens": 144019553, "loss_tokens": 71419, "loss": 2.301681144023299, "lr": 0.00029958704415366155, "optimizer_lrs": [0.019972469610244105, 0.00029958704415366155], "gradient_norm": 0.5432724952697754, "seconds": 0.3298740849999149, "nonpadding_tokens_per_second": 216503.82145059507, "ranks": [{"compute_seconds": 0.32565571800000725, "nonpadding_tokens": 12075, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3242031169999109, "nonpadding_tokens": 10912, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3237465209999755, "nonpadding_tokens": 5778, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32737774500003525, "nonpadding_tokens": 6069, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32747336399995675, "nonpadding_tokens": 10084, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32493616100009604, "nonpadding_tokens": 3007, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32742879499994615, "nonpadding_tokens": 18276, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3237136909999663, "nonpadding_tokens": 5218, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043060799998784205, "prefetch": {"data_wait_seconds": 0.01263870900102404, "consumed_batches": 1348, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1898, "tokens": 144074383, "loss_tokens": 54830, "loss": 2.218652539668065, "lr": 0.00029958667911749956, "optimizer_lrs": [0.019972445274499973, 0.00029958667911749956], "gradient_norm": 0.45099884271621704, "seconds": 0.2842705409999553, "nonpadding_tokens_per_second": 192879.64136955232, "ranks": [{"compute_seconds": 0.2805339259999755, "nonpadding_tokens": 10808, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.281856093999977, "nonpadding_tokens": 5506, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2771513010000035, "nonpadding_tokens": 17302, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28032687699999315, "nonpadding_tokens": 3139, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2808912399999599, "nonpadding_tokens": 3350, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27792806100001144, "nonpadding_tokens": 2927, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28076978299998245, "nonpadding_tokens": 7423, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2767728060000536, "nonpadding_tokens": 4375, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004324559999986377, "prefetch": {"data_wait_seconds": 0.012649161001107245, "consumed_batches": 1349, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1899, "tokens": 144142804, "loss_tokens": 68421, "loss": 2.6069456471697285, "lr": 0.00029958622337207145, "optimizer_lrs": [0.019972414891471432, 0.00029958622337207145], "gradient_norm": 0.41590291261672974, "seconds": 0.3368238460000157, "nonpadding_tokens_per_second": 203135.85517337988, "ranks": [{"compute_seconds": 0.33347359899994444, "nonpadding_tokens": 3611, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3328245440000046, "nonpadding_tokens": 8624, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3328782290000163, "nonpadding_tokens": 14739, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33349885300003734, "nonpadding_tokens": 2265, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3287598969999408, "nonpadding_tokens": 2561, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33128781100003835, "nonpadding_tokens": 10076, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3333299570000463, "nonpadding_tokens": 17404, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3344339529999161, "nonpadding_tokens": 9141, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004225640000186104, "prefetch": {"data_wait_seconds": 0.012659211001121662, "consumed_batches": 1350, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1900, "tokens": 144195501, "loss_tokens": 52697, "loss": 2.308601236787673, "lr": 0.00029958587219170715, "optimizer_lrs": [0.019972391479447144, 0.00029958587219170715], "gradient_norm": 0.5337174534797668, "seconds": 0.3198734920000561, "nonpadding_tokens_per_second": 164743.25418622297, "ranks": [{"compute_seconds": 0.3138839989999269, "nonpadding_tokens": 9105, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3129136569999673, "nonpadding_tokens": 3702, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3159398700000793, "nonpadding_tokens": 2693, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3135260090000429, "nonpadding_tokens": 7460, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3169742989999804, "nonpadding_tokens": 4821, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3146012789999304, "nonpadding_tokens": 9711, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3120833879999054, "nonpadding_tokens": 3697, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31754398800001127, "nonpadding_tokens": 11508, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003939409999702548, "prefetch": {"data_wait_seconds": 0.012668015001125355, "consumed_batches": 1351, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1901, "tokens": 144262757, "loss_tokens": 67256, "loss": 2.4572517228946116, "lr": 0.00029958542377203005, "optimizer_lrs": [0.019972361584802006, 0.00029958542377203005], "gradient_norm": 0.37545308470726013, "seconds": 0.35007041699998354, "nonpadding_tokens_per_second": 192121.3468317809, "ranks": [{"compute_seconds": 0.3472455800000489, "nonpadding_tokens": 12397, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34328706300004797, "nonpadding_tokens": 12569, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3463312469999664, "nonpadding_tokens": 10632, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3447282270000187, "nonpadding_tokens": 7735, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3431343269999161, "nonpadding_tokens": 5489, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34557002799999736, "nonpadding_tokens": 9408, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34684060999995836, "nonpadding_tokens": 3374, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34771545600005993, "nonpadding_tokens": 5652, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039671699994414666, "prefetch": {"data_wait_seconds": 0.0126762180011184, "consumed_batches": 1352, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1902, "tokens": 144308481, "loss_tokens": 45724, "loss": 2.4765500270645613, "lr": 0.00029958511877551427, "optimizer_lrs": [0.019972341251700953, 0.00029958511877551427], "gradient_norm": 0.41564008593559265, "seconds": 0.29554229699999723, "nonpadding_tokens_per_second": 154712.20351244826, "ranks": [{"compute_seconds": 0.29226060500002404, "nonpadding_tokens": 7467, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2919968850000032, "nonpadding_tokens": 9292, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2915470789999972, "nonpadding_tokens": 4564, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29227729000001546, "nonpadding_tokens": 5578, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29295082700002695, "nonpadding_tokens": 5689, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2899083229999633, "nonpadding_tokens": 5905, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29214405300001545, "nonpadding_tokens": 1726, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29311122900003284, "nonpadding_tokens": 5503, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041470400003618124, "prefetch": {"data_wait_seconds": 0.012686456001077318, "consumed_batches": 1353, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1903, "tokens": 144385548, "loss_tokens": 77067, "loss": 2.5357568333398213, "lr": 0.0002995846044558154, "optimizer_lrs": [0.01997230696372103, 0.0002995846044558154], "gradient_norm": 0.42425721883773804, "seconds": 0.2648760270000139, "nonpadding_tokens_per_second": 290954.98325333896, "ranks": [{"compute_seconds": 0.2437629859999788, "nonpadding_tokens": 9697, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2492254229999844, "nonpadding_tokens": 20205, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24823258099991108, "nonpadding_tokens": 2527, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2488457129999233, "nonpadding_tokens": 2605, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2528529260000596, "nonpadding_tokens": 8378, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25019381599997814, "nonpadding_tokens": 2247, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2526598649999414, "nonpadding_tokens": 19211, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2532524040000226, "nonpadding_tokens": 12197, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042155800008458755, "prefetch": {"data_wait_seconds": 0.012694999000999019, "consumed_batches": 1354, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1904, "tokens": 144448350, "loss_tokens": 62802, "loss": 2.4362271504092226, "lr": 0.00029958418510077936, "optimizer_lrs": [0.019972279006718625, 0.00029958418510077936], "gradient_norm": 0.3877045214176178, "seconds": 0.2839526919999571, "nonpadding_tokens_per_second": 221170.64486224166, "ranks": [{"compute_seconds": 0.28042837899999995, "nonpadding_tokens": 7436, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27940033699997, "nonpadding_tokens": 5798, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27623309299997345, "nonpadding_tokens": 9994, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2807669999999689, "nonpadding_tokens": 3799, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2810877970000547, "nonpadding_tokens": 7316, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2784010320000334, "nonpadding_tokens": 7698, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2755481689999897, "nonpadding_tokens": 10290, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2815633939999316, "nonpadding_tokens": 10471, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004240949999712029, "prefetch": {"data_wait_seconds": 0.012702466001044854, "consumed_batches": 1355, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1905, "tokens": 144533244, "loss_tokens": 84894, "loss": 2.63584624649563, "lr": 0.0002995836178925076, "optimizer_lrs": [0.01997224119283384, 0.0002995836178925076], "gradient_norm": 0.39610880613327026, "seconds": 0.26296401199999764, "nonpadding_tokens_per_second": 322835.05014367047, "ranks": [{"compute_seconds": 0.259609730999955, "nonpadding_tokens": 4047, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25760457499995937, "nonpadding_tokens": 10981, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25960916499991527, "nonpadding_tokens": 14924, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2603044029999637, "nonpadding_tokens": 17958, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2568180370000164, "nonpadding_tokens": 2956, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2583096689999138, "nonpadding_tokens": 5587, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26051101899997775, "nonpadding_tokens": 16174, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25757462699993994, "nonpadding_tokens": 12267, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004017859999976281, "prefetch": {"data_wait_seconds": 0.012713538001051461, "consumed_batches": 1356, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1906, "tokens": 144624967, "loss_tokens": 91723, "loss": 2.7289060881676352, "lr": 0.00029958300462333185, "optimizer_lrs": [0.019972200308222124, 0.00029958300462333185], "gradient_norm": 0.6772341132164001, "seconds": 0.35340761000009024, "nonpadding_tokens_per_second": 259538.8367555995, "ranks": [{"compute_seconds": 0.33646049000003586, "nonpadding_tokens": 16428, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3481127210000068, "nonpadding_tokens": 10781, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3450809170000184, "nonpadding_tokens": 12963, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3485716180000509, "nonpadding_tokens": 17282, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34910218400000304, "nonpadding_tokens": 9070, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34696772899997086, "nonpadding_tokens": 8637, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3488557000000583, "nonpadding_tokens": 3716, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34928259599996636, "nonpadding_tokens": 12846, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003525549999494615, "prefetch": {"data_wait_seconds": 0.012722451001039303, "consumed_batches": 1357, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1907, "tokens": 144671328, "loss_tokens": 46361, "loss": 2.14660557904273, "lr": 0.0002995826944776026, "optimizer_lrs": [0.019972179631840174, 0.0002995826944776026], "gradient_norm": 0.4286772608757019, "seconds": 0.28763702799994917, "nonpadding_tokens_per_second": 161178.83125954212, "ranks": [{"compute_seconds": 0.2839844439999979, "nonpadding_tokens": 1678, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2827016899999535, "nonpadding_tokens": 2748, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27965253999991546, "nonpadding_tokens": 3697, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28051222600004166, "nonpadding_tokens": 11625, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28516164299992397, "nonpadding_tokens": 8301, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2809897729999875, "nonpadding_tokens": 8532, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2837335210000447, "nonpadding_tokens": 5608, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2842859520000047, "nonpadding_tokens": 4172, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038569599996662873, "prefetch": {"data_wait_seconds": 0.012732099000984931, "consumed_batches": 1358, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1908, "tokens": 144734598, "loss_tokens": 63270, "loss": 2.2621542595226805, "lr": 0.00029958227102832636, "optimizer_lrs": [0.019972151401888426, 0.00029958227102832636], "gradient_norm": 0.43766963481903076, "seconds": 0.34476272699998844, "nonpadding_tokens_per_second": 183517.51812196948, "ranks": [{"compute_seconds": 0.3418335259999594, "nonpadding_tokens": 13364, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.338435486000094, "nonpadding_tokens": 20966, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33770221599991146, "nonpadding_tokens": 2843, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3423583659999849, "nonpadding_tokens": 4564, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.336424743000066, "nonpadding_tokens": 6779, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3395436199999722, "nonpadding_tokens": 6562, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3420875100000558, "nonpadding_tokens": 4225, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3370806120000225, "nonpadding_tokens": 3967, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003971470000578847, "prefetch": {"data_wait_seconds": 0.01274174300097286, "consumed_batches": 1359, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1909, "tokens": 144790654, "loss_tokens": 56056, "loss": 2.4261781745040674, "lr": 0.0002995818956813077, "optimizer_lrs": [0.019972126378753847, 0.0002995818956813077], "gradient_norm": 0.5231409668922424, "seconds": 0.2961324869999089, "nonpadding_tokens_per_second": 189293.65220242535, "ranks": [{"compute_seconds": 0.29296299299994644, "nonpadding_tokens": 15553, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29139799699999, "nonpadding_tokens": 3981, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29232936699997936, "nonpadding_tokens": 8425, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2928602969999474, "nonpadding_tokens": 8285, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2872843229999944, "nonpadding_tokens": 1637, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29079152400004205, "nonpadding_tokens": 8214, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2875039079999624, "nonpadding_tokens": 3681, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2937146929999699, "nonpadding_tokens": 6280, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041981200001828256, "prefetch": {"data_wait_seconds": 0.012751846000995215, "consumed_batches": 1360, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1910, "tokens": 144855457, "loss_tokens": 64803, "loss": 2.159993316281654, "lr": 0.0002995814615552592, "optimizer_lrs": [0.019972097437017285, 0.0002995814615552592], "gradient_norm": 0.40974730253219604, "seconds": 0.291239128999905, "nonpadding_tokens_per_second": 222507.87599361737, "ranks": [{"compute_seconds": 0.28780272300002707, "nonpadding_tokens": 4733, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2883179680000012, "nonpadding_tokens": 16217, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28718947999993816, "nonpadding_tokens": 11865, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2860310250000566, "nonpadding_tokens": 14512, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2840486789999659, "nonpadding_tokens": 3249, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2853642640000089, "nonpadding_tokens": 4195, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28800541800001156, "nonpadding_tokens": 7671, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2889037099999996, "nonpadding_tokens": 2361, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000395268000033866, "prefetch": {"data_wait_seconds": 0.01276087800097514, "consumed_batches": 1361, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1911, "tokens": 144914964, "loss_tokens": 59507, "loss": 2.160172595240896, "lr": 0.00029958106270989976, "optimizer_lrs": [0.01997207084732665, 0.00029958106270989976], "gradient_norm": 0.4226689338684082, "seconds": 0.3232463470000084, "nonpadding_tokens_per_second": 184091.79423765754, "ranks": [{"compute_seconds": 0.3206650299999865, "nonpadding_tokens": 5483, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.317582692999963, "nonpadding_tokens": 2329, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3197481010000729, "nonpadding_tokens": 7469, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.318209564999961, "nonpadding_tokens": 5671, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3167081160000862, "nonpadding_tokens": 6401, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31966605299999173, "nonpadding_tokens": 3588, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32010379100006503, "nonpadding_tokens": 6605, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32099080499995125, "nonpadding_tokens": 21961, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003461889999698542, "prefetch": {"data_wait_seconds": 0.012770128001037051, "consumed_batches": 1362, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1912, "tokens": 144988445, "loss_tokens": 73481, "loss": 2.5988070045317837, "lr": 0.0002995805699421905, "optimizer_lrs": [0.019972037996146032, 0.0002995805699421905], "gradient_norm": 0.3745401203632355, "seconds": 0.2561623530000361, "nonpadding_tokens_per_second": 286853.2363925844, "ranks": [{"compute_seconds": 0.2531020080000417, "nonpadding_tokens": 6506, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2538955799999485, "nonpadding_tokens": 5889, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25269584499994835, "nonpadding_tokens": 2263, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25116968999998335, "nonpadding_tokens": 7488, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24972846700006812, "nonpadding_tokens": 16995, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25090639200004716, "nonpadding_tokens": 12685, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2533678460000601, "nonpadding_tokens": 15286, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2505469710000625, "nonpadding_tokens": 6369, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000345058999982939, "prefetch": {"data_wait_seconds": 0.01277894500101695, "consumed_batches": 1363, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1913, "tokens": 145048722, "loss_tokens": 60277, "loss": 2.3880465393101846, "lr": 0.00029958016550534004, "optimizer_lrs": [0.019972011033689337, 0.00029958016550534004], "gradient_norm": 0.4345119595527649, "seconds": 0.2942452490000278, "nonpadding_tokens_per_second": 204852.92525485877, "ranks": [{"compute_seconds": 0.2919117250000909, "nonpadding_tokens": 4782, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28693419900002937, "nonpadding_tokens": 3270, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28943886199999724, "nonpadding_tokens": 6328, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28995310499999505, "nonpadding_tokens": 3150, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28603335699995114, "nonpadding_tokens": 19116, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2885683130000416, "nonpadding_tokens": 6305, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28558380800006944, "nonpadding_tokens": 11894, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29087302199991427, "nonpadding_tokens": 5432, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003900720000729052, "prefetch": {"data_wait_seconds": 0.012787945000994227, "consumed_batches": 1364, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1914, "tokens": 145110440, "loss_tokens": 61718, "loss": 2.3065596746492107, "lr": 0.00029957975119827547, "optimizer_lrs": [0.019971983413218364, 0.00029957975119827547], "gradient_norm": 0.4459489583969116, "seconds": 0.256363212999986, "nonpadding_tokens_per_second": 240744.3692009094, "ranks": [{"compute_seconds": 0.25248450000003686, "nonpadding_tokens": 4152, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2538644710000426, "nonpadding_tokens": 4125, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25245946299992283, "nonpadding_tokens": 9472, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25350930199999766, "nonpadding_tokens": 10022, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24956031300007453, "nonpadding_tokens": 11670, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2507356579999396, "nonpadding_tokens": 5950, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25337058799993883, "nonpadding_tokens": 5507, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25033499800008485, "nonpadding_tokens": 10820, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005715769999596887, "prefetch": {"data_wait_seconds": 0.012794659001045261, "consumed_batches": 1365, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1915, "tokens": 145175016, "loss_tokens": 64576, "loss": 2.0760339561137267, "lr": 0.0002995793174873327, "optimizer_lrs": [0.019971954499155516, 0.0002995793174873327], "gradient_norm": 0.46060505509376526, "seconds": 0.32137950000003457, "nonpadding_tokens_per_second": 200933.7870025719, "ranks": [{"compute_seconds": 0.31839086299999053, "nonpadding_tokens": 8926, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.314700728000048, "nonpadding_tokens": 8075, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31401010100000804, "nonpadding_tokens": 5412, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31473128799996175, "nonpadding_tokens": 12149, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.318490665000013, "nonpadding_tokens": 15974, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3159972880000623, "nonpadding_tokens": 2482, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3131140299999515, "nonpadding_tokens": 9812, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31901502799996706, "nonpadding_tokens": 1746, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003624080000008689, "prefetch": {"data_wait_seconds": 0.012803333001059514, "consumed_batches": 1366, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1916, "tokens": 145229141, "loss_tokens": 54125, "loss": 2.5695046189376445, "lr": 0.00029957895379625947, "optimizer_lrs": [0.019971930253083966, 0.00029957895379625947], "gradient_norm": 0.4227336049079895, "seconds": 0.24639824600001248, "nonpadding_tokens_per_second": 219664.71303532436, "ranks": [{"compute_seconds": 0.24188588100003017, "nonpadding_tokens": 9376, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24049603299999944, "nonpadding_tokens": 11868, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2431557380000413, "nonpadding_tokens": 2781, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24377441500007535, "nonpadding_tokens": 3208, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24405936100004055, "nonpadding_tokens": 5007, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24135073800005102, "nonpadding_tokens": 11968, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23982297399993513, "nonpadding_tokens": 5966, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24089961000004223, "nonpadding_tokens": 3951, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004027429999950982, "prefetch": {"data_wait_seconds": 0.012811145001094815, "consumed_batches": 1367, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1917, "tokens": 145305469, "loss_tokens": 76328, "loss": 2.482524024604339, "lr": 0.000299578440646252, "optimizer_lrs": [0.01997189604308347, 0.000299578440646252], "gradient_norm": 0.5179747939109802, "seconds": 0.27722481699993295, "nonpadding_tokens_per_second": 275328.885869617, "ranks": [{"compute_seconds": 0.27039687599994977, "nonpadding_tokens": 2104, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27411787999994885, "nonpadding_tokens": 11478, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27334936500005824, "nonpadding_tokens": 3006, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2740922020000198, "nonpadding_tokens": 11360, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27042138100000557, "nonpadding_tokens": 14626, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27329755199991723, "nonpadding_tokens": 14452, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2699627900000223, "nonpadding_tokens": 10605, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27491038300001946, "nonpadding_tokens": 8697, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037785899996833905, "prefetch": {"data_wait_seconds": 0.01281900200110897, "consumed_batches": 1368, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1918, "tokens": 145371462, "loss_tokens": 65993, "loss": 2.524702904095889, "lr": 0.00029957799672651587, "optimizer_lrs": [0.01997186644843439, 0.00029957799672651587], "gradient_norm": 0.4102567732334137, "seconds": 0.4280872489999865, "nonpadding_tokens_per_second": 154157.8268312357, "ranks": [{"compute_seconds": 0.42366277500002525, "nonpadding_tokens": 22141, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.42188419100000374, "nonpadding_tokens": 3848, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.42442343099992286, "nonpadding_tokens": 4190, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4232639450000306, "nonpadding_tokens": 4594, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.42138176399998883, "nonpadding_tokens": 6457, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.42279055100004825, "nonpadding_tokens": 13619, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4213435550000213, "nonpadding_tokens": 5686, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.42576089400006367, "nonpadding_tokens": 5458, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000410680999948454, "prefetch": {"data_wait_seconds": 0.012828015001062454, "consumed_batches": 1369, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1919, "tokens": 145448179, "loss_tokens": 76717, "loss": 2.5606559660831367, "lr": 0.0002995774803757199, "optimizer_lrs": [0.019971832025047997, 0.0002995774803757199], "gradient_norm": 0.7111839056015015, "seconds": 0.3046704779999345, "nonpadding_tokens_per_second": 251803.19571368673, "ranks": [{"compute_seconds": 0.30219600999998875, "nonpadding_tokens": 15721, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30204248600000483, "nonpadding_tokens": 17002, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2988171299999749, "nonpadding_tokens": 6585, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3018039429999817, "nonpadding_tokens": 4458, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.297673992, "nonpadding_tokens": 4976, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3002808649999906, "nonpadding_tokens": 9896, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3020729190000111, "nonpadding_tokens": 15277, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2986544820000745, "nonpadding_tokens": 2802, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000350615000002108, "prefetch": {"data_wait_seconds": 0.012836695001126941, "consumed_batches": 1370, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1920, "tokens": 145485163, "loss_tokens": 36984, "loss": 2.322513620754921, "lr": 0.00029957723133891404, "optimizer_lrs": [0.019971815422594272, 0.00029957723133891404], "gradient_norm": 0.39086541533470154, "seconds": 0.2831287090000387, "nonpadding_tokens_per_second": 130626.10333872904, "ranks": [{"compute_seconds": 0.2801046540000698, "nonpadding_tokens": 4356, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27775828200003616, "nonpadding_tokens": 8168, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28024783999990177, "nonpadding_tokens": 3968, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.278352735999988, "nonpadding_tokens": 5248, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2758942150001076, "nonpadding_tokens": 4960, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2789890789999845, "nonpadding_tokens": 2905, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28076828999996906, "nonpadding_tokens": 5129, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27656256099999155, "nonpadding_tokens": 2250, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003905359999407665, "prefetch": {"data_wait_seconds": 0.012845990001167706, "consumed_batches": 1371, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1921, "tokens": 145546157, "loss_tokens": 60994, "loss": 2.4401390915499883, "lr": 0.0002995768204674711, "optimizer_lrs": [0.01997178803116474, 0.0002995768204674711], "gradient_norm": 0.38140150904655457, "seconds": 0.25054808500010495, "nonpadding_tokens_per_second": 243442.29172605512, "ranks": [{"compute_seconds": 0.24714546000006976, "nonpadding_tokens": 4806, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2482463620000317, "nonpadding_tokens": 6461, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24493842999993376, "nonpadding_tokens": 2159, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24710591200005183, "nonpadding_tokens": 12279, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24784773600003973, "nonpadding_tokens": 9562, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24652090300003238, "nonpadding_tokens": 16726, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24385542600009558, "nonpadding_tokens": 3492, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24432928600003834, "nonpadding_tokens": 5509, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035193799999433395, "prefetch": {"data_wait_seconds": 0.01285352600116596, "consumed_batches": 1372, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1922, "tokens": 145619584, "loss_tokens": 73427, "loss": 2.4462289076225368, "lr": 0.0002995763255798032, "optimizer_lrs": [0.01997175503865355, 0.0002995763255798032], "gradient_norm": 0.5663114786148071, "seconds": 0.2889682879999782, "nonpadding_tokens_per_second": 254100.54683926265, "ranks": [{"compute_seconds": 0.24700591099997382, "nonpadding_tokens": 10375, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25270331799993073, "nonpadding_tokens": 9402, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2553410770000255, "nonpadding_tokens": 4721, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25429380500008847, "nonpadding_tokens": 11652, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25654722000001584, "nonpadding_tokens": 9117, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25525393100008387, "nonpadding_tokens": 7348, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25262813099993764, "nonpadding_tokens": 17010, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25704724599995643, "nonpadding_tokens": 3802, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.010990460000016355, "prefetch": {"data_wait_seconds": 0.012862310001153787, "consumed_batches": 1373, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1923, "tokens": 145676315, "loss_tokens": 56731, "loss": 2.58259868942906, "lr": 0.0002995759430231012, "optimizer_lrs": [0.019971729534873413, 0.0002995759430231012], "gradient_norm": 0.4228748381137848, "seconds": 0.2587196590000076, "nonpadding_tokens_per_second": 219275.9538230465, "ranks": [{"compute_seconds": 0.25505801099996006, "nonpadding_tokens": 8638, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2521277229999441, "nonpadding_tokens": 7399, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25266424800008735, "nonpadding_tokens": 2445, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25368121199994675, "nonpadding_tokens": 2886, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2511560160000954, "nonpadding_tokens": 14744, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2528409929999498, "nonpadding_tokens": 7564, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2553832539999803, "nonpadding_tokens": 8693, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25618982100002086, "nonpadding_tokens": 4362, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005301239999653262, "prefetch": {"data_wait_seconds": 0.012869476001199587, "consumed_batches": 1374, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1924, "tokens": 145785076, "loss_tokens": 108761, "loss": 2.682357704048326, "lr": 0.00029957520912818255, "optimizer_lrs": [0.019971680608545504, 0.00029957520912818255], "gradient_norm": 0.45035138726234436, "seconds": 0.2891447209999569, "nonpadding_tokens_per_second": 376147.2788569991, "ranks": [{"compute_seconds": 0.2867534770000475, "nonpadding_tokens": 12296, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28650355000002037, "nonpadding_tokens": 17794, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2835548130000234, "nonpadding_tokens": 19472, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2864714690000483, "nonpadding_tokens": 17011, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2824020960000553, "nonpadding_tokens": 2840, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28541136199999073, "nonpadding_tokens": 3780, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28651420599999256, "nonpadding_tokens": 22186, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28334100500001114, "nonpadding_tokens": 13382, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004027369999448638, "prefetch": {"data_wait_seconds": 0.012877569001105371, "consumed_batches": 1375, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1925, "tokens": 145837365, "loss_tokens": 52289, "loss": 2.44630652479489, "lr": 0.00029957485606819717, "optimizer_lrs": [0.019971657071213147, 0.00029957485606819717], "gradient_norm": 0.4760110676288605, "seconds": 0.30423182600009113, "nonpadding_tokens_per_second": 171872.22220460372, "ranks": [{"compute_seconds": 0.29889922899997146, "nonpadding_tokens": 3257, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29762022599993543, "nonpadding_tokens": 7761, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30038696200006143, "nonpadding_tokens": 12470, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29928207900002235, "nonpadding_tokens": 10370, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29727886700004547, "nonpadding_tokens": 5571, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29986521099999663, "nonpadding_tokens": 5713, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3016923760000054, "nonpadding_tokens": 2995, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.301943266999956, "nonpadding_tokens": 4152, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003503920000866856, "prefetch": {"data_wait_seconds": 0.012887180001101228, "consumed_batches": 1376, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1926, "tokens": 145892816, "loss_tokens": 55451, "loss": 2.400694701177616, "lr": 0.00029957448149812776, "optimizer_lrs": [0.019971632099875184, 0.00029957448149812776], "gradient_norm": 0.5497865676879883, "seconds": 0.2645780450000075, "nonpadding_tokens_per_second": 209582.7716921804, "ranks": [{"compute_seconds": 0.2614336840000533, "nonpadding_tokens": 6617, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26147148199993353, "nonpadding_tokens": 10922, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26105324799993923, "nonpadding_tokens": 4295, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2591889660000106, "nonpadding_tokens": 4137, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2618959650000079, "nonpadding_tokens": 10624, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26032412699998986, "nonpadding_tokens": 3834, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26160770899991803, "nonpadding_tokens": 7688, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2621954100000039, "nonpadding_tokens": 7334, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040136399991297367, "prefetch": {"data_wait_seconds": 0.012896066001147233, "consumed_batches": 1377, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1927, "tokens": 145936251, "loss_tokens": 43435, "loss": 2.6512720156555774, "lr": 0.00029957418798081385, "optimizer_lrs": [0.019971612532054258, 0.00029957418798081385], "gradient_norm": 0.3918117880821228, "seconds": 0.2550958349999064, "nonpadding_tokens_per_second": 170269.34210829408, "ranks": [{"compute_seconds": 0.25277546400002393, "nonpadding_tokens": 3808, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25153707499998745, "nonpadding_tokens": 3413, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2515235070000017, "nonpadding_tokens": 5113, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24965379700006451, "nonpadding_tokens": 2496, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25271706099999847, "nonpadding_tokens": 3627, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24952859099994384, "nonpadding_tokens": 10545, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2523808270000245, "nonpadding_tokens": 7978, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2489989370000103, "nonpadding_tokens": 6455, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003929090000838187, "prefetch": {"data_wait_seconds": 0.012905344001140406, "consumed_batches": 1378, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1928, "tokens": 145987346, "loss_tokens": 51095, "loss": 2.448967915158039, "lr": 0.00029957384257077864, "optimizer_lrs": [0.019971589504718577, 0.00029957384257077864], "gradient_norm": 0.4152452349662781, "seconds": 0.32597624000004544, "nonpadding_tokens_per_second": 156744.55291585939, "ranks": [{"compute_seconds": 0.3233180409999932, "nonpadding_tokens": 9804, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3194599419999804, "nonpadding_tokens": 6284, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3206525869999268, "nonpadding_tokens": 10987, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.320902546999946, "nonpadding_tokens": 3701, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32352386699994895, "nonpadding_tokens": 5154, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3203962719999254, "nonpadding_tokens": 5705, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31950517999996464, "nonpadding_tokens": 4556, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3236119640000652, "nonpadding_tokens": 4904, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040008799999213807, "prefetch": {"data_wait_seconds": 0.012915020001130983, "consumed_batches": 1379, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1929, "tokens": 146052586, "loss_tokens": 65240, "loss": 2.135624952099939, "lr": 0.0002995734013351405, "optimizer_lrs": [0.01997156008900937, 0.0002995734013351405], "gradient_norm": 0.567430317401886, "seconds": 0.326731562999953, "nonpadding_tokens_per_second": 199674.617906472, "ranks": [{"compute_seconds": 0.324399470000003, "nonpadding_tokens": 2017, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32077006399993024, "nonpadding_tokens": 5789, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3208938689999741, "nonpadding_tokens": 4786, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3242927640000062, "nonpadding_tokens": 5671, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32056767699998545, "nonpadding_tokens": 15286, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32326463599997624, "nonpadding_tokens": 10327, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31992733000004137, "nonpadding_tokens": 8753, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3211764279999443, "nonpadding_tokens": 12611, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037506400008169294, "prefetch": {"data_wait_seconds": 0.01292491700117182, "consumed_batches": 1380, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1930, "tokens": 146112083, "loss_tokens": 59497, "loss": 2.64464673639007, "lr": 0.00029957299874222473, "optimizer_lrs": [0.01997153324948165, 0.00029957299874222473], "gradient_norm": 0.41520529985427856, "seconds": 0.3091077579999819, "nonpadding_tokens_per_second": 192479.80181721444, "ranks": [{"compute_seconds": 0.30468135499995697, "nonpadding_tokens": 13178, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3054557940000677, "nonpadding_tokens": 12214, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30563921700002084, "nonpadding_tokens": 6046, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30653550099998483, "nonpadding_tokens": 5101, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3067817499999137, "nonpadding_tokens": 6511, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3043329150000318, "nonpadding_tokens": 5581, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3012930590000451, "nonpadding_tokens": 7127, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3018788829999721, "nonpadding_tokens": 3739, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003656439999986105, "prefetch": {"data_wait_seconds": 0.012933707001209882, "consumed_batches": 1381, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1931, "tokens": 146172690, "loss_tokens": 60607, "loss": 2.522804193410002, "lr": 0.00029957258844345646, "optimizer_lrs": [0.01997150589623043, 0.00029957258844345646], "gradient_norm": 0.36588364839553833, "seconds": 0.3107146749999856, "nonpadding_tokens_per_second": 195056.76711279506, "ranks": [{"compute_seconds": 0.3080000380000456, "nonpadding_tokens": 12950, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3082530190000625, "nonpadding_tokens": 6837, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30724206899992623, "nonpadding_tokens": 9280, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30543281199993544, "nonpadding_tokens": 8473, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3040292099999533, "nonpadding_tokens": 6496, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3053094650000503, "nonpadding_tokens": 3427, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3076628389999314, "nonpadding_tokens": 5241, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3082739110000148, "nonpadding_tokens": 7903, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004356819999884465, "prefetch": {"data_wait_seconds": 0.01295791400112023, "consumed_batches": 1382, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1932, "tokens": 146262557, "loss_tokens": 89867, "loss": 2.191779066565035, "lr": 0.00029957197969751576, "optimizer_lrs": [0.019971465313167718, 0.00029957197969751576], "gradient_norm": 0.5008428692817688, "seconds": 0.46370180000008077, "nonpadding_tokens_per_second": 193803.4314293892, "ranks": [{"compute_seconds": 0.46007509599996865, "nonpadding_tokens": 24956, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.46072541099999853, "nonpadding_tokens": 15987, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4600884929999438, "nonpadding_tokens": 16209, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.46051504199999727, "nonpadding_tokens": 1971, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4613672579999957, "nonpadding_tokens": 7523, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.45968045899996923, "nonpadding_tokens": 2638, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4608256669999946, "nonpadding_tokens": 9909, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4578381899999613, "nonpadding_tokens": 10674, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000347877999956836, "prefetch": {"data_wait_seconds": 0.012966231001044548, "consumed_batches": 1383, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1933, "tokens": 146361062, "loss_tokens": 98505, "loss": 2.011739886300188, "lr": 0.0002995713119421568, "optimizer_lrs": [0.019971420796143786, 0.0002995713119421568], "gradient_norm": 0.5381115078926086, "seconds": 0.4244918639999469, "nonpadding_tokens_per_second": 232053.91752811623, "ranks": [{"compute_seconds": 0.4208569049999369, "nonpadding_tokens": 9958, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4164638479999212, "nonpadding_tokens": 2715, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4196379900000693, "nonpadding_tokens": 8600, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4203187429999389, "nonpadding_tokens": 24988, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.41622983100000965, "nonpadding_tokens": 10911, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41954797699997926, "nonpadding_tokens": 11823, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4205018860000109, "nonpadding_tokens": 10856, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.42138209899997037, "nonpadding_tokens": 18654, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038989399990896345, "prefetch": {"data_wait_seconds": 0.01297564800104567, "consumed_batches": 1384, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1934, "tokens": 146420977, "loss_tokens": 59915, "loss": 2.3434404468830845, "lr": 0.0002995709055303148, "optimizer_lrs": [0.019971393702020987, 0.0002995709055303148], "gradient_norm": 0.44311612844467163, "seconds": 0.2700901380000005, "nonpadding_tokens_per_second": 221833.34957605854, "ranks": [{"compute_seconds": 0.2674140599999646, "nonpadding_tokens": 3139, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26343566200000623, "nonpadding_tokens": 9464, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26391649699996833, "nonpadding_tokens": 10203, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2648232880000023, "nonpadding_tokens": 2106, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2631430909999608, "nonpadding_tokens": 9836, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2647449180000194, "nonpadding_tokens": 8884, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2667428030000565, "nonpadding_tokens": 12110, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26759494400005224, "nonpadding_tokens": 4173, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047389999997449195, "prefetch": {"data_wait_seconds": 0.012985597000991334, "consumed_batches": 1385, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1935, "tokens": 146476765, "loss_tokens": 55788, "loss": 2.641339479368323, "lr": 0.00029957052693965086, "optimizer_lrs": [0.019971368462643392, 0.00029957052693965086], "gradient_norm": 0.4574364721775055, "seconds": 0.2735555410000643, "nonpadding_tokens_per_second": 203936.64773175583, "ranks": [{"compute_seconds": 0.20449655300001268, "nonpadding_tokens": 1955, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2203010749999521, "nonpadding_tokens": 21545, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2178892659999292, "nonpadding_tokens": 6728, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21819012099990687, "nonpadding_tokens": 10215, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21650598299993362, "nonpadding_tokens": 6115, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21615126999995482, "nonpadding_tokens": 1290, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2183079930000531, "nonpadding_tokens": 3360, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21884469899998749, "nonpadding_tokens": 4580, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.011510530000009567, "prefetch": {"data_wait_seconds": 0.012995869001045435, "consumed_batches": 1386, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1936, "tokens": 146566770, "loss_tokens": 90005, "loss": 2.5327308205099714, "lr": 0.0002995699157929004, "optimizer_lrs": [0.019971327719526696, 0.0002995699157929004], "gradient_norm": 0.3732618987560272, "seconds": 0.2735393140000042, "nonpadding_tokens_per_second": 329038.6258700591, "ranks": [{"compute_seconds": 0.2686449430000266, "nonpadding_tokens": 4707, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26825265400009357, "nonpadding_tokens": 11048, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2678239400000848, "nonpadding_tokens": 14562, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2709422530000438, "nonpadding_tokens": 4589, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.266850411000064, "nonpadding_tokens": 10661, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2688127080000413, "nonpadding_tokens": 8105, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2669091469999785, "nonpadding_tokens": 15486, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2677445159999934, "nonpadding_tokens": 20847, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003954470000735455, "prefetch": {"data_wait_seconds": 0.013004983000953985, "consumed_batches": 1387, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1937, "tokens": 146656233, "loss_tokens": 89463, "loss": 2.649410929658071, "lr": 0.0002995693078964853, "optimizer_lrs": [0.01997128719309902, 0.0002995693078964853], "gradient_norm": 0.4067382216453552, "seconds": 0.3320610420000776, "nonpadding_tokens_per_second": 269417.33200963424, "ranks": [{"compute_seconds": 0.3270369890000211, "nonpadding_tokens": 19707, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3258191589999342, "nonpadding_tokens": 10654, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32813312599989786, "nonpadding_tokens": 3106, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3269943120000107, "nonpadding_tokens": 12048, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3252604430000474, "nonpadding_tokens": 8723, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3267224159999387, "nonpadding_tokens": 14951, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3249790859999848, "nonpadding_tokens": 3551, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32972189899999194, "nonpadding_tokens": 16723, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004328129999748853, "prefetch": {"data_wait_seconds": 0.013013593000962942, "consumed_batches": 1388, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1938, "tokens": 146741462, "loss_tokens": 85229, "loss": 2.3127141289936524, "lr": 0.0002995687283712055, "optimizer_lrs": [0.019971248558080367, 0.0002995687283712055], "gradient_norm": 0.41527363657951355, "seconds": 0.3653481599999395, "nonpadding_tokens_per_second": 233281.59090773607, "ranks": [{"compute_seconds": 0.36305431199991745, "nonpadding_tokens": 28919, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36283926300006897, "nonpadding_tokens": 4626, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36189528600004905, "nonpadding_tokens": 6283, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3598885459999792, "nonpadding_tokens": 13718, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36301291300003413, "nonpadding_tokens": 4846, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3604412050000292, "nonpadding_tokens": 15577, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3588635849999946, "nonpadding_tokens": 5360, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3595522810001057, "nonpadding_tokens": 5900, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003435190000118382, "prefetch": {"data_wait_seconds": 0.01302298400101165, "consumed_batches": 1389, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1939, "tokens": 146808008, "loss_tokens": 66546, "loss": 2.6325915532113124, "lr": 0.0002995682756128843, "optimizer_lrs": [0.019971218374192287, 0.0002995682756128843], "gradient_norm": 1.764430284500122, "seconds": 0.3786897100000033, "nonpadding_tokens_per_second": 175726.98238882548, "ranks": [{"compute_seconds": 0.3756507519999559, "nonpadding_tokens": 13630, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37222910100001627, "nonpadding_tokens": 8721, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37453250199996546, "nonpadding_tokens": 20420, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3727302060000284, "nonpadding_tokens": 10076, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.37170061500000884, "nonpadding_tokens": 1382, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37292894699999124, "nonpadding_tokens": 3239, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3754999459999908, "nonpadding_tokens": 3889, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37606790099994214, "nonpadding_tokens": 5189, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003970449999997072, "prefetch": {"data_wait_seconds": 0.01303223800103126, "consumed_batches": 1390, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1940, "tokens": 146858775, "loss_tokens": 50767, "loss": 2.2724224459294424, "lr": 0.0002995679300505088, "optimizer_lrs": [0.019971195336700588, 0.0002995679300505088], "gradient_norm": 0.3900536894798279, "seconds": 0.2993043259999695, "nonpadding_tokens_per_second": 169616.6596670079, "ranks": [{"compute_seconds": 0.29314308400000755, "nonpadding_tokens": 10847, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29298875200004204, "nonpadding_tokens": 5465, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2959682260000136, "nonpadding_tokens": 7086, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29635200300003817, "nonpadding_tokens": 9035, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2970046639999282, "nonpadding_tokens": 2470, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2941110709999748, "nonpadding_tokens": 2833, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29672259500000564, "nonpadding_tokens": 3091, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29244613800005936, "nonpadding_tokens": 9940, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035465199994177965, "prefetch": {"data_wait_seconds": 0.013041510001087886, "consumed_batches": 1391, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1941, "tokens": 146909851, "loss_tokens": 51076, "loss": 2.407115591226799, "lr": 0.00029956758224553877, "optimizer_lrs": [0.019971172149702586, 0.00029956758224553877], "gradient_norm": 1.285521149635315, "seconds": 0.27718108600004143, "nonpadding_tokens_per_second": 184269.42738795807, "ranks": [{"compute_seconds": 0.256897720999973, "nonpadding_tokens": 15678, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25772132700001293, "nonpadding_tokens": 1224, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2600728560000789, "nonpadding_tokens": 4391, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.258505375000027, "nonpadding_tokens": 7705, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2611605119999467, "nonpadding_tokens": 4651, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2584414589999824, "nonpadding_tokens": 10273, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2573324360000697, "nonpadding_tokens": 3039, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26152528499994787, "nonpadding_tokens": 4115, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004304839999349497, "prefetch": {"data_wait_seconds": 0.013049407001133773, "consumed_batches": 1392, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1942, "tokens": 146976364, "loss_tokens": 66513, "loss": 2.2190309601130607, "lr": 0.00029956712911200314, "optimizer_lrs": [0.01997114194080021, 0.00029956712911200314], "gradient_norm": 0.42394888401031494, "seconds": 0.32210137499998837, "nonpadding_tokens_per_second": 206497.10048584052, "ranks": [{"compute_seconds": 0.3191825640000161, "nonpadding_tokens": 17088, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31762867700001607, "nonpadding_tokens": 3569, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31859890199996244, "nonpadding_tokens": 1987, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3190430940001079, "nonpadding_tokens": 11243, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3195015879999801, "nonpadding_tokens": 8500, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31657386300003054, "nonpadding_tokens": 8023, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3137989709999829, "nonpadding_tokens": 5873, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3198178369999596, "nonpadding_tokens": 10230, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003548370000316936, "prefetch": {"data_wait_seconds": 0.013056694001150504, "consumed_batches": 1393, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1943, "tokens": 147051064, "loss_tokens": 74700, "loss": 2.7773217871485945, "lr": 0.0002995666199203884, "optimizer_lrs": [0.01997110799469256, 0.0002995666199203884], "gradient_norm": 0.4136093854904175, "seconds": 0.27654891500003487, "nonpadding_tokens_per_second": 270114.9632063846, "ranks": [{"compute_seconds": 0.27365240300002824, "nonpadding_tokens": 13173, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2711463040000126, "nonpadding_tokens": 2045, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2734349669999574, "nonpadding_tokens": 13383, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2740140779999365, "nonpadding_tokens": 7736, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2702542130000438, "nonpadding_tokens": 5226, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27185091499995906, "nonpadding_tokens": 7207, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27423271600002863, "nonpadding_tokens": 8675, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2706120599999622, "nonpadding_tokens": 17255, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036186399995585816, "prefetch": {"data_wait_seconds": 0.013065504001204431, "consumed_batches": 1394, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1944, "tokens": 147105577, "loss_tokens": 54513, "loss": 2.495116990442647, "lr": 0.00029956624814460065, "optimizer_lrs": [0.019971083209640045, 0.00029956624814460065], "gradient_norm": 0.4193356931209564, "seconds": 0.19950238900003114, "nonpadding_tokens_per_second": 273244.8482107725, "ranks": [{"compute_seconds": 0.17185231700000259, "nonpadding_tokens": 1915, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.18486386899996887, "nonpadding_tokens": 9411, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.18395122899994476, "nonpadding_tokens": 4750, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.1829402729999856, "nonpadding_tokens": 15099, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1811722260000579, "nonpadding_tokens": 5325, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.18229698900006497, "nonpadding_tokens": 2175, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.18103520099998605, "nonpadding_tokens": 6505, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.18187821999993048, "nonpadding_tokens": 9333, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.005093794999993406, "prefetch": {"data_wait_seconds": 0.01307484800118175, "consumed_batches": 1395, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1945, "tokens": 147150960, "loss_tokens": 45383, "loss": 2.370184388978252, "lr": 0.00029956593851354033, "optimizer_lrs": [0.019971062567569357, 0.00029956593851354033], "gradient_norm": 0.40339457988739014, "seconds": 0.39058661899991876, "nonpadding_tokens_per_second": 116191.89647664156, "ranks": [{"compute_seconds": 0.38797035800007507, "nonpadding_tokens": 5483, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3875700510000115, "nonpadding_tokens": 1186, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.38496093100002327, "nonpadding_tokens": 2133, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.385431949000008, "nonpadding_tokens": 12948, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.388113328999907, "nonpadding_tokens": 1677, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38527273400006834, "nonpadding_tokens": 4739, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3836550949999946, "nonpadding_tokens": 4162, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3882612300000119, "nonpadding_tokens": 13055, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037968000003729685, "prefetch": {"data_wait_seconds": 0.013084780001193508, "consumed_batches": 1396, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1946, "tokens": 147202875, "loss_tokens": 51915, "loss": 2.2960507861408073, "lr": 0.00029956558418186045, "optimizer_lrs": [0.019971038945457365, 0.00029956558418186045], "gradient_norm": 0.47657325863838196, "seconds": 0.24884761100008745, "nonpadding_tokens_per_second": 208621.65319313336, "ranks": [{"compute_seconds": 0.24553192299993043, "nonpadding_tokens": 2220, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24296324600004482, "nonpadding_tokens": 3463, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24552611800004343, "nonpadding_tokens": 9675, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24384116199996697, "nonpadding_tokens": 9363, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24647940699992432, "nonpadding_tokens": 12130, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24530907299993032, "nonpadding_tokens": 8033, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24647301699997115, "nonpadding_tokens": 4960, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24308176199997433, "nonpadding_tokens": 2071, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036703199998555647, "prefetch": {"data_wait_seconds": 0.013093361001168091, "consumed_batches": 1397, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1947, "tokens": 147244829, "loss_tokens": 41954, "loss": 2.505225403120084, "lr": 0.000299565297730822, "optimizer_lrs": [0.019971019848721468, 0.000299565297730822], "gradient_norm": 0.4333195686340332, "seconds": 0.3406751049999457, "nonpadding_tokens_per_second": 123149.59145607862, "ranks": [{"compute_seconds": 0.33789970999998786, "nonpadding_tokens": 2968, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33809239900006105, "nonpadding_tokens": 5995, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3351297890000069, "nonpadding_tokens": 7929, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.33577114399997754, "nonpadding_tokens": 3132, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33786851399997886, "nonpadding_tokens": 3951, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33671617800007425, "nonpadding_tokens": 7444, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3376860690000285, "nonpadding_tokens": 5063, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3383132690000821, "nonpadding_tokens": 5472, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038175700001374935, "prefetch": {"data_wait_seconds": 0.013102811001161285, "consumed_batches": 1398, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1948, "tokens": 147290047, "loss_tokens": 45218, "loss": 3.076231395683135, "lr": 0.00029956498888849376, "optimizer_lrs": [0.01997099925923292, 0.00029956498888849376], "gradient_norm": 0.4746808409690857, "seconds": 0.22742387299990696, "nonpadding_tokens_per_second": 198826.9718721152, "ranks": [{"compute_seconds": 0.22055187499995554, "nonpadding_tokens": 3367, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2214138819999789, "nonpadding_tokens": 11014, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21791988999996192, "nonpadding_tokens": 13333, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21863711700007116, "nonpadding_tokens": 5194, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2245696659999794, "nonpadding_tokens": 2721, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21977953999999045, "nonpadding_tokens": 2294, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2242486050000707, "nonpadding_tokens": 3943, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22513578400003098, "nonpadding_tokens": 3352, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035447299990210013, "prefetch": {"data_wait_seconds": 0.013112749001109592, "consumed_batches": 1399, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1949, "tokens": 147382536, "loss_tokens": 92489, "loss": 2.2823827360550983, "lr": 0.0002995643568406664, "optimizer_lrs": [0.019970957122711092, 0.0002995643568406664], "gradient_norm": 0.6151940822601318, "seconds": 0.317838791999975, "nonpadding_tokens_per_second": 290993.42914695974, "ranks": [{"compute_seconds": 0.3039886540000225, "nonpadding_tokens": 1718, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3026048390000824, "nonpadding_tokens": 3220, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30218917799993505, "nonpadding_tokens": 17320, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30575826300002973, "nonpadding_tokens": 18816, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30624401499994747, "nonpadding_tokens": 21882, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.304361343000096, "nonpadding_tokens": 16737, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30152030399995056, "nonpadding_tokens": 3248, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30653904400003285, "nonpadding_tokens": 9548, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042040699997869524, "prefetch": {"data_wait_seconds": 0.013122146001094848, "consumed_batches": 1400, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1950, "tokens": 147469905, "loss_tokens": 87369, "loss": 2.2992440825693325, "lr": 0.00029956375936094715, "optimizer_lrs": [0.01997091729072981, 0.00029956375936094715], "gradient_norm": 0.42261791229248047, "seconds": 0.3825144900000623, "nonpadding_tokens_per_second": 228407.03367860854, "ranks": [{"compute_seconds": 0.3781787110000323, "nonpadding_tokens": 19015, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37885606200006805, "nonpadding_tokens": 12540, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3786885990000428, "nonpadding_tokens": 12297, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3794572959999414, "nonpadding_tokens": 3063, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3752289030001066, "nonpadding_tokens": 4319, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37679154900001777, "nonpadding_tokens": 11535, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37482279500000004, "nonpadding_tokens": 10976, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.380157281000038, "nonpadding_tokens": 13624, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003559680000080334, "prefetch": {"data_wait_seconds": 0.013130503001050897, "consumed_batches": 1401, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1951, "tokens": 147628492, "loss_tokens": 158587, "loss": 1.67913038269215, "lr": 0.0002995626738069894, "optimizer_lrs": [0.019970844920465962, 0.0002995626738069894], "gradient_norm": 0.821989893913269, "seconds": 0.4203463990000955, "nonpadding_tokens_per_second": 377276.9324948207, "ranks": [{"compute_seconds": 0.4172607790000029, "nonpadding_tokens": 20760, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4142735929999617, "nonpadding_tokens": 21674, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.41713537300006465, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.4157451390000233, "nonpadding_tokens": 18598, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.417964434000055, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4165001809999467, "nonpadding_tokens": 18156, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.41376478999995925, "nonpadding_tokens": 5177, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4143046140000024, "nonpadding_tokens": 8686, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004342070000120657, "prefetch": {"data_wait_seconds": 0.01314905600106897, "consumed_batches": 1402, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1952, "tokens": 147669675, "loss_tokens": 41183, "loss": 2.4378998919457056, "lr": 0.0002995623916823324, "optimizer_lrs": [0.019970826112155497, 0.0002995623916823324], "gradient_norm": 0.46800360083580017, "seconds": 0.24990619600009722, "nonpadding_tokens_per_second": 164793.833282885, "ranks": [{"compute_seconds": 0.24645097700010865, "nonpadding_tokens": 5957, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24664779800002634, "nonpadding_tokens": 5702, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24618928899997172, "nonpadding_tokens": 1198, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24416620200008765, "nonpadding_tokens": 4892, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24242384000001493, "nonpadding_tokens": 9591, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24457870999992792, "nonpadding_tokens": 3130, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24699319799992736, "nonpadding_tokens": 3681, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24751657500007695, "nonpadding_tokens": 7032, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004919509999581351, "prefetch": {"data_wait_seconds": 0.01315677000111748, "consumed_batches": 1403, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1953, "tokens": 147754792, "loss_tokens": 85117, "loss": 2.467070326726741, "lr": 0.0002995618082994166, "optimizer_lrs": [0.01997078721996111, 0.0002995618082994166], "gradient_norm": 0.41983523964881897, "seconds": 0.3255957840000292, "nonpadding_tokens_per_second": 261419.23262738675, "ranks": [{"compute_seconds": 0.3069327359999079, "nonpadding_tokens": 6901, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31269447099998615, "nonpadding_tokens": 13183, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31329672700007904, "nonpadding_tokens": 17583, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3160680800000364, "nonpadding_tokens": 6176, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31661501899998257, "nonpadding_tokens": 18081, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3153053409999984, "nonpadding_tokens": 4119, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31614066400004504, "nonpadding_tokens": 11473, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3125892630000635, "nonpadding_tokens": 7601, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004327089999378586, "prefetch": {"data_wait_seconds": 0.013166753001158327, "consumed_batches": 1404, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1954, "tokens": 147842003, "loss_tokens": 87211, "loss": 2.4035702563323436, "lr": 0.0002995612101620445, "optimizer_lrs": [0.019970747344136303, 0.0002995612101620445], "gradient_norm": 0.3780020773410797, "seconds": 0.28492821699990145, "nonpadding_tokens_per_second": 306080.6013467952, "ranks": [{"compute_seconds": 0.21501996800009238, "nonpadding_tokens": 4087, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22308392899992668, "nonpadding_tokens": 4289, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22221769299994776, "nonpadding_tokens": 6325, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2206237359999932, "nonpadding_tokens": 14568, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22295631700001195, "nonpadding_tokens": 18600, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22133281000003535, "nonpadding_tokens": 16056, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22275224500003787, "nonpadding_tokens": 17943, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2233059200000298, "nonpadding_tokens": 5343, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.04303549600001588, "prefetch": {"data_wait_seconds": 0.01317476700110376, "consumed_batches": 1405, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1955, "tokens": 147935045, "loss_tokens": 93042, "loss": 2.6036759608563873, "lr": 0.00029956057158366244, "optimizer_lrs": [0.019970704772244163, 0.00029956057158366244], "gradient_norm": 0.4192701280117035, "seconds": 0.4217019039999741, "nonpadding_tokens_per_second": 220634.52670587352, "ranks": [{"compute_seconds": 0.4126145960000258, "nonpadding_tokens": 12592, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.41236277699999846, "nonpadding_tokens": 4865, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4150974750000387, "nonpadding_tokens": 21652, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.41581907099998716, "nonpadding_tokens": 12467, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.41651939199994104, "nonpadding_tokens": 9002, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4131882140000016, "nonpadding_tokens": 1624, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.415916601000049, "nonpadding_tokens": 3202, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.41663585999992847, "nonpadding_tokens": 27638, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.002836339999930715, "prefetch": {"data_wait_seconds": 0.013183133001078318, "consumed_batches": 1406, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1956, "tokens": 148003444, "loss_tokens": 68399, "loss": 2.1308564909574703, "lr": 0.00029956010184277606, "optimizer_lrs": [0.019970673456185074, 0.00029956010184277606], "gradient_norm": 0.3898438513278961, "seconds": 0.3516333830000349, "nonpadding_tokens_per_second": 194517.9363132118, "ranks": [{"compute_seconds": 0.34706107599993175, "nonpadding_tokens": 26580, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3451120949999904, "nonpadding_tokens": 13064, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.341854718000036, "nonpadding_tokens": 12052, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3478703250000308, "nonpadding_tokens": 3941, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3482143859999951, "nonpadding_tokens": 5951, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3455207460000338, "nonpadding_tokens": 1517, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34788269099999525, "nonpadding_tokens": 2357, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34889792800004216, "nonpadding_tokens": 2937, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005154290000746187, "prefetch": {"data_wait_seconds": 0.013189443001081713, "consumed_batches": 1407, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1957, "tokens": 148072304, "loss_tokens": 68860, "loss": 2.4635814514957888, "lr": 0.00029955962868283276, "optimizer_lrs": [0.019970641912188853, 0.00029955962868283276], "gradient_norm": 0.4207189083099365, "seconds": 0.2675378550000005, "nonpadding_tokens_per_second": 257384.13728404854, "ranks": [{"compute_seconds": 0.2647494899999856, "nonpadding_tokens": 3696, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26158446699992055, "nonpadding_tokens": 10884, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26220759700004237, "nonpadding_tokens": 3215, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26258074299994405, "nonpadding_tokens": 4321, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26512491599999066, "nonpadding_tokens": 12535, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2636915030000182, "nonpadding_tokens": 3815, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26106134700000894, "nonpadding_tokens": 12235, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2614438609999752, "nonpadding_tokens": 18159, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035577199992076203, "prefetch": {"data_wait_seconds": 0.013198788001048456, "consumed_batches": 1408, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1958, "tokens": 148139059, "loss_tokens": 66755, "loss": 2.3762714403415472, "lr": 0.0002995591697446525, "optimizer_lrs": [0.019970611316310168, 0.0002995591697446525], "gradient_norm": 0.38550081849098206, "seconds": 0.2969161889999441, "nonpadding_tokens_per_second": 224827.75433983686, "ranks": [{"compute_seconds": 0.2931093309999824, "nonpadding_tokens": 7418, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2922622109999793, "nonpadding_tokens": 9085, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29330473599998186, "nonpadding_tokens": 11137, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2895219800000177, "nonpadding_tokens": 6488, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28830000699997527, "nonpadding_tokens": 3883, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29120551899995917, "nonpadding_tokens": 11280, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29373885500001506, "nonpadding_tokens": 13672, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2944923729999118, "nonpadding_tokens": 3792, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000363199000048553, "prefetch": {"data_wait_seconds": 0.013208869001076096, "consumed_batches": 1409, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1959, "tokens": 148251722, "loss_tokens": 112663, "loss": 2.5897632763196436, "lr": 0.00029955839464943667, "optimizer_lrs": [0.01997055964329578, 0.00029955839464943667], "gradient_norm": 0.5300695896148682, "seconds": 0.3890101470000218, "nonpadding_tokens_per_second": 289614.55342190264, "ranks": [{"compute_seconds": 0.38400041099998816, "nonpadding_tokens": 20669, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3826118160000078, "nonpadding_tokens": 18745, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.38503609700001107, "nonpadding_tokens": 20661, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.38572080900007677, "nonpadding_tokens": 19645, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.38666760800003885, "nonpadding_tokens": 11971, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3853382380000312, "nonpadding_tokens": 11160, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.382312082999988, "nonpadding_tokens": 4116, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3828050780000467, "nonpadding_tokens": 5696, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003952470000285757, "prefetch": {"data_wait_seconds": 0.013219508001114377, "consumed_batches": 1410, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1960, "tokens": 148308045, "loss_tokens": 56323, "loss": 2.6261879028105746, "lr": 0.000299558006905472, "optimizer_lrs": [0.019970533793698137, 0.000299558006905472], "gradient_norm": 0.4942428767681122, "seconds": 0.2903276410000899, "nonpadding_tokens_per_second": 193998.06303659032, "ranks": [{"compute_seconds": 0.28802860899997995, "nonpadding_tokens": 6148, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28526092000004155, "nonpadding_tokens": 20178, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2876271300000326, "nonpadding_tokens": 3099, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.287907096999902, "nonpadding_tokens": 7909, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28463044899990564, "nonpadding_tokens": 2587, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28696571000000404, "nonpadding_tokens": 2956, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2848182779999888, "nonpadding_tokens": 10809, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2851967799999784, "nonpadding_tokens": 2637, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036851099991963565, "prefetch": {"data_wait_seconds": 0.013228355001118075, "consumed_batches": 1411, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1961, "tokens": 148367047, "loss_tokens": 59002, "loss": 2.4682558430222703, "lr": 0.0002995576005363054, "optimizer_lrs": [0.01997050670242036, 0.0002995576005363054], "gradient_norm": 0.458732008934021, "seconds": 0.35181146900004023, "nonpadding_tokens_per_second": 167709.14310355598, "ranks": [{"compute_seconds": 0.34388469400005306, "nonpadding_tokens": 14079, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3456482850000384, "nonpadding_tokens": 6186, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3482242769999857, "nonpadding_tokens": 4538, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3429061819999788, "nonpadding_tokens": 2968, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34937151600001926, "nonpadding_tokens": 11808, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34692888699999, "nonpadding_tokens": 2414, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3490761940000766, "nonpadding_tokens": 10024, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34184940000000097, "nonpadding_tokens": 6985, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003743889999441308, "prefetch": {"data_wait_seconds": 0.013237705001145628, "consumed_batches": 1412, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1962, "tokens": 148431622, "loss_tokens": 64575, "loss": 2.3413271873790165, "lr": 0.00029955715557011364, "optimizer_lrs": [0.019970477038007577, 0.00029955715557011364], "gradient_norm": 0.426498681306839, "seconds": 0.21926409000002423, "nonpadding_tokens_per_second": 294507.8694828363, "ranks": [{"compute_seconds": 0.21265246600000864, "nonpadding_tokens": 2852, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21328445200003898, "nonpadding_tokens": 11201, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21154032999993433, "nonpadding_tokens": 10757, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.21629169599998477, "nonpadding_tokens": 10610, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21132360199999312, "nonpadding_tokens": 6552, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21412622400009695, "nonpadding_tokens": 6417, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21633109499998682, "nonpadding_tokens": 7020, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21697272800008705, "nonpadding_tokens": 9166, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003718909999861353, "prefetch": {"data_wait_seconds": 0.01324655000105679, "consumed_batches": 1413, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1963, "tokens": 148504028, "loss_tokens": 72406, "loss": 2.3954167731265366, "lr": 0.0002995566563773886, "optimizer_lrs": [0.019970443758492575, 0.0002995566563773886], "gradient_norm": 0.5469437837600708, "seconds": 0.27550581799994234, "nonpadding_tokens_per_second": 262811.14687754126, "ranks": [{"compute_seconds": 0.272694400999967, "nonpadding_tokens": 11910, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26974844199992276, "nonpadding_tokens": 12606, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27249380799992196, "nonpadding_tokens": 7580, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26844688599999245, "nonpadding_tokens": 4870, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2668013620000238, "nonpadding_tokens": 6043, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26989765300004365, "nonpadding_tokens": 7426, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27315821700005927, "nonpadding_tokens": 17334, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2676164230000495, "nonpadding_tokens": 4637, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033460199995261064, "prefetch": {"data_wait_seconds": 0.013254072001089412, "consumed_batches": 1414, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1964, "tokens": 148565625, "loss_tokens": 61597, "loss": 2.446995693783788, "lr": 0.0002995562314847562, "optimizer_lrs": [0.01997041543231708, 0.0002995562314847562], "gradient_norm": 0.40065425634384155, "seconds": 0.3015369529999816, "nonpadding_tokens_per_second": 204276.7872632969, "ranks": [{"compute_seconds": 0.2985440219999873, "nonpadding_tokens": 3457, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29500614999994923, "nonpadding_tokens": 2117, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2927795949999563, "nonpadding_tokens": 3965, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.293706745999998, "nonpadding_tokens": 12887, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.292176851000022, "nonpadding_tokens": 5924, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2947823150000204, "nonpadding_tokens": 7331, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.298570447999964, "nonpadding_tokens": 10622, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2991679149999982, "nonpadding_tokens": 15294, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038004099997124285, "prefetch": {"data_wait_seconds": 0.013261607001098241, "consumed_batches": 1415, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1965, "tokens": 148626369, "loss_tokens": 60744, "loss": 2.0501093729421838, "lr": 0.00029955581227709846, "optimizer_lrs": [0.0199703874851399, 0.00029955581227709846], "gradient_norm": 0.4225468337535858, "seconds": 0.3867841869999893, "nonpadding_tokens_per_second": 157048.81957855655, "ranks": [{"compute_seconds": 0.382672824999986, "nonpadding_tokens": 10575, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3817710309999711, "nonpadding_tokens": 5084, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.38323331499998403, "nonpadding_tokens": 2467, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3841709879999371, "nonpadding_tokens": 1561, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.38433228400003827, "nonpadding_tokens": 17298, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3817904349999708, "nonpadding_tokens": 9682, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37828970800001116, "nonpadding_tokens": 4138, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37886378799998965, "nonpadding_tokens": 9939, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004507439999770213, "prefetch": {"data_wait_seconds": 0.013270081001067524, "consumed_batches": 1416, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1966, "tokens": 148677387, "loss_tokens": 51018, "loss": 2.5682902357991297, "lr": 0.0002995554600380363, "optimizer_lrs": [0.019970364002535757, 0.0002995554600380363], "gradient_norm": 0.44086953997612, "seconds": 0.2515854890000355, "nonpadding_tokens_per_second": 202785.94048797782, "ranks": [{"compute_seconds": 0.21500770000000102, "nonpadding_tokens": 6541, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22931478500004232, "nonpadding_tokens": 12297, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22828711900001508, "nonpadding_tokens": 7287, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2288909499999363, "nonpadding_tokens": 9992, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23277647899999465, "nonpadding_tokens": 1329, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2300944689999369, "nonpadding_tokens": 4877, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22690605799994046, "nonpadding_tokens": 6132, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23288931900003718, "nonpadding_tokens": 2563, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.006738493000057133, "prefetch": {"data_wait_seconds": 0.013279548000994623, "consumed_batches": 1417, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1967, "tokens": 148738232, "loss_tokens": 60845, "loss": 2.6356353747226557, "lr": 0.0002995550397690562, "optimizer_lrs": [0.01997033598460375, 0.0002995550397690562], "gradient_norm": 0.4874374568462372, "seconds": 0.3453381790000094, "nonpadding_tokens_per_second": 176189.6126752853, "ranks": [{"compute_seconds": 0.3429109300000164, "nonpadding_tokens": 8559, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3403634729999112, "nonpadding_tokens": 13334, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3374083659999769, "nonpadding_tokens": 4933, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.337880049999967, "nonpadding_tokens": 3503, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3362800840000091, "nonpadding_tokens": 9339, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.340170976999957, "nonpadding_tokens": 7993, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3429193659999328, "nonpadding_tokens": 9237, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33694242800004304, "nonpadding_tokens": 3947, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041088199998284836, "prefetch": {"data_wait_seconds": 0.0132896820010302, "consumed_batches": 1418, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1968, "tokens": 148800422, "loss_tokens": 62190, "loss": 2.544438917028461, "lr": 0.0002995546100050219, "optimizer_lrs": [0.019970307333668127, 0.0002995546100050219], "gradient_norm": 0.40056079626083374, "seconds": 0.38485003799996775, "nonpadding_tokens_per_second": 161595.41083377835, "ranks": [{"compute_seconds": 0.38089695199994367, "nonpadding_tokens": 9027, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.38200151400008053, "nonpadding_tokens": 2243, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3788365850000446, "nonpadding_tokens": 5220, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3819191840000258, "nonpadding_tokens": 9799, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.38248941300003025, "nonpadding_tokens": 9872, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3804966970000123, "nonpadding_tokens": 6395, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3777438399999937, "nonpadding_tokens": 4975, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3783708649999653, "nonpadding_tokens": 14659, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040601500006687274, "prefetch": {"data_wait_seconds": 0.013298207001028572, "consumed_batches": 1419, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1969, "tokens": 148902449, "loss_tokens": 102027, "loss": 2.4786886988248207, "lr": 0.00029955390449879117, "optimizer_lrs": [0.019970260299919412, 0.00029955390449879117], "gradient_norm": 0.39712926745414734, "seconds": 0.28323078099992927, "nonpadding_tokens_per_second": 360225.67758984317, "ranks": [{"compute_seconds": 0.2801815520000446, "nonpadding_tokens": 11038, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27686748199994327, "nonpadding_tokens": 10213, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27477848900002755, "nonpadding_tokens": 17951, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2796959080000079, "nonpadding_tokens": 20235, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2742363219999788, "nonpadding_tokens": 6970, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2768285109999624, "nonpadding_tokens": 3187, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27396705599994675, "nonpadding_tokens": 7820, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2807731089999379, "nonpadding_tokens": 24613, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043120199995883013, "prefetch": {"data_wait_seconds": 0.01330829400099276, "consumed_batches": 1420, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1970, "tokens": 148959567, "loss_tokens": 57118, "loss": 2.5250254954655276, "lr": 0.00029955350929031124, "optimizer_lrs": [0.01997023395268742, 0.00029955350929031124], "gradient_norm": 1.0686585903167725, "seconds": 0.2631872320000639, "nonpadding_tokens_per_second": 217024.20579424663, "ranks": [{"compute_seconds": 0.2605012319999105, "nonpadding_tokens": 5381, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2599981649999563, "nonpadding_tokens": 11055, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25680258800002775, "nonpadding_tokens": 5020, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26006050599994524, "nonpadding_tokens": 6040, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25696058599999105, "nonpadding_tokens": 8568, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25875529699999333, "nonpadding_tokens": 3616, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25570891600000323, "nonpadding_tokens": 8926, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26084248900008333, "nonpadding_tokens": 8512, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003689339999937147, "prefetch": {"data_wait_seconds": 0.013317105001036111, "consumed_batches": 1421, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1971, "tokens": 149048121, "loss_tokens": 88554, "loss": 2.210961673103417, "lr": 0.00029955289622579884, "optimizer_lrs": [0.019970193081719923, 0.00029955289622579884], "gradient_norm": 0.4264616072177887, "seconds": 0.4238470740000366, "nonpadding_tokens_per_second": 208929.1289999382, "ranks": [{"compute_seconds": 0.42047853700000815, "nonpadding_tokens": 13533, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4204147709999688, "nonpadding_tokens": 14276, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.41721192100010285, "nonpadding_tokens": 6562, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.42037882299996454, "nonpadding_tokens": 11878, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.42110251299993706, "nonpadding_tokens": 9885, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41876480500002344, "nonpadding_tokens": 22710, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.41618868900002326, "nonpadding_tokens": 2178, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4214161249999506, "nonpadding_tokens": 7532, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003494589999490927, "prefetch": {"data_wait_seconds": 0.01332605900104511, "consumed_batches": 1422, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1972, "tokens": 149126526, "loss_tokens": 78405, "loss": 2.5503419743638798, "lr": 0.00029955235307293034, "optimizer_lrs": [0.019970156871528692, 0.00029955235307293034], "gradient_norm": 0.42889800667762756, "seconds": 0.24281437100000858, "nonpadding_tokens_per_second": 322900.9867788972, "ranks": [{"compute_seconds": 0.23812355400002616, "nonpadding_tokens": 8239, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2388792790000025, "nonpadding_tokens": 2915, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2359554180000032, "nonpadding_tokens": 4679, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23939198800007944, "nonpadding_tokens": 21659, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24029345799999646, "nonpadding_tokens": 9293, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23739107699998385, "nonpadding_tokens": 10879, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23508538299995507, "nonpadding_tokens": 5965, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24030633400002444, "nonpadding_tokens": 14776, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004514040000458408, "prefetch": {"data_wait_seconds": 0.01333504900112814, "consumed_batches": 1423, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1973, "tokens": 149201500, "loss_tokens": 74974, "loss": 2.2125126293781845, "lr": 0.00029955183338053375, "optimizer_lrs": [0.019970122225368918, 0.00029955183338053375], "gradient_norm": 0.42603960633277893, "seconds": 0.3127661700000317, "nonpadding_tokens_per_second": 239712.62620887803, "ranks": [{"compute_seconds": 0.30946192900000824, "nonpadding_tokens": 11631, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3074553210000204, "nonpadding_tokens": 7511, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3055543549999129, "nonpadding_tokens": 12046, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.31037454299996625, "nonpadding_tokens": 10530, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30440348500007985, "nonpadding_tokens": 4402, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3083125949999612, "nonpadding_tokens": 2177, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30442085000004226, "nonpadding_tokens": 17120, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30525341500003833, "nonpadding_tokens": 9557, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042148500006078393, "prefetch": {"data_wait_seconds": 0.0133446820012324, "consumed_batches": 1424, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1974, "tokens": 149262886, "loss_tokens": 61386, "loss": 2.437772354445639, "lr": 0.0002995514076510846, "optimizer_lrs": [0.01997009384340564, 0.0002995514076510846], "gradient_norm": 0.7748199105262756, "seconds": 0.26135929000008673, "nonpadding_tokens_per_second": 234872.0797335332, "ranks": [{"compute_seconds": 0.25640399400003844, "nonpadding_tokens": 7643, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25680823400000463, "nonpadding_tokens": 3041, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25338181799997983, "nonpadding_tokens": 10082, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25808494000000337, "nonpadding_tokens": 2548, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2585296270000299, "nonpadding_tokens": 11320, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25580742299996473, "nonpadding_tokens": 1776, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25815283600002203, "nonpadding_tokens": 16425, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25889971700007663, "nonpadding_tokens": 8551, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040990599995893717, "prefetch": {"data_wait_seconds": 0.01335423800124147, "consumed_batches": 1425, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1975, "tokens": 149336674, "loss_tokens": 73788, "loss": 2.696736766140836, "lr": 0.00029955089564318087, "optimizer_lrs": [0.019970059709545393, 0.00029955089564318087], "gradient_norm": 0.4103092551231384, "seconds": 0.2640108919999875, "nonpadding_tokens_per_second": 279488.46898333076, "ranks": [{"compute_seconds": 0.25658287500004917, "nonpadding_tokens": 5155, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25750947999995333, "nonpadding_tokens": 6454, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.259876069000029, "nonpadding_tokens": 16663, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2571462580000343, "nonpadding_tokens": 10954, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2551742350000268, "nonpadding_tokens": 2538, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2582695969999804, "nonpadding_tokens": 17682, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25506618999997954, "nonpadding_tokens": 3526, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2615169580000156, "nonpadding_tokens": 10816, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039953499992861907, "prefetch": {"data_wait_seconds": 0.013362964001160549, "consumed_batches": 1426, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1976, "tokens": 149389580, "loss_tokens": 52906, "loss": 2.2178211710392017, "lr": 0.0002995505283540298, "optimizer_lrs": [0.019970035223601988, 0.0002995505283540298], "gradient_norm": 0.563768744468689, "seconds": 0.3642045039999857, "nonpadding_tokens_per_second": 145264.5407152957, "ranks": [{"compute_seconds": 0.3601797239999769, "nonpadding_tokens": 10721, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.361250623999922, "nonpadding_tokens": 3072, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3605649450000783, "nonpadding_tokens": 3494, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3587979469999709, "nonpadding_tokens": 3448, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35730974400007653, "nonpadding_tokens": 6923, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3600027400000272, "nonpadding_tokens": 3745, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35706698899991807, "nonpadding_tokens": 13312, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36180374800005666, "nonpadding_tokens": 8191, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003735760000154187, "prefetch": {"data_wait_seconds": 0.013371508001185362, "consumed_batches": 1427, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1977, "tokens": 149465577, "loss_tokens": 75997, "loss": 2.4793848770346196, "lr": 0.0002995500004980305, "optimizer_lrs": [0.019970000033202033, 0.0002995500004980305], "gradient_norm": 0.41733992099761963, "seconds": 0.33200966499998685, "nonpadding_tokens_per_second": 228899.9628971735, "ranks": [{"compute_seconds": 0.3295178579999174, "nonpadding_tokens": 4826, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32508384000004753, "nonpadding_tokens": 14644, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32574400799990144, "nonpadding_tokens": 5671, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.32873084699997435, "nonpadding_tokens": 1169, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32888526099998217, "nonpadding_tokens": 18277, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32629758500002026, "nonpadding_tokens": 18507, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32442155100000036, "nonpadding_tokens": 8508, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3255495199999814, "nonpadding_tokens": 4395, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005031489999964833, "prefetch": {"data_wait_seconds": 0.01337973900110967, "consumed_batches": 1428, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1978, "tokens": 149511281, "loss_tokens": 45704, "loss": 2.5929628355833185, "lr": 0.00029954968290065413, "optimizer_lrs": [0.01996997886004361, 0.00029954968290065413], "gradient_norm": 0.5908236503601074, "seconds": 0.30674159000000145, "nonpadding_tokens_per_second": 148998.38003708524, "ranks": [{"compute_seconds": 0.3023607619999211, "nonpadding_tokens": 9564, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.304318569999964, "nonpadding_tokens": 9731, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3030638679999811, "nonpadding_tokens": 5935, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3036849860000075, "nonpadding_tokens": 5704, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30006018599999607, "nonpadding_tokens": 3661, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3013745730000892, "nonpadding_tokens": 3573, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.303853836999906, "nonpadding_tokens": 6322, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3008710289999499, "nonpadding_tokens": 1214, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004357749999144289, "prefetch": {"data_wait_seconds": 0.01338787600111857, "consumed_batches": 1429, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1979, "tokens": 149574793, "loss_tokens": 63512, "loss": 2.353571224729185, "lr": 0.00029954924136966636, "optimizer_lrs": [0.019969949424644425, 0.00029954924136966636], "gradient_norm": 0.42874231934547424, "seconds": 0.26620564899997134, "nonpadding_tokens_per_second": 238582.46524290263, "ranks": [{"compute_seconds": 0.26374638100003267, "nonpadding_tokens": 6694, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26390759599996727, "nonpadding_tokens": 7466, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2607829229999652, "nonpadding_tokens": 7316, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26289969899994503, "nonpadding_tokens": 7177, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2599705949999134, "nonpadding_tokens": 3930, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26113601199995173, "nonpadding_tokens": 16701, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2633722599999828, "nonpadding_tokens": 4412, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2603463740000507, "nonpadding_tokens": 9816, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039586200000485405, "prefetch": {"data_wait_seconds": 0.013396572001127538, "consumed_batches": 1430, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1980, "tokens": 149645406, "loss_tokens": 70613, "loss": 2.483397842819311, "lr": 0.0002995487502194591, "optimizer_lrs": [0.019969916681297276, 0.0002995487502194591], "gradient_norm": 0.3740196228027344, "seconds": 0.25135099099998115, "nonpadding_tokens_per_second": 280933.84362270246, "ranks": [{"compute_seconds": 0.24513033699997777, "nonpadding_tokens": 5776, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24453697600006308, "nonpadding_tokens": 9637, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24483656000006704, "nonpadding_tokens": 9760, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24521788399999878, "nonpadding_tokens": 6694, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24322471900006803, "nonpadding_tokens": 8023, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24510308499998246, "nonpadding_tokens": 7297, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24768282100001215, "nonpadding_tokens": 10300, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24901739799997813, "nonpadding_tokens": 13126, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003763440000739138, "prefetch": {"data_wait_seconds": 0.013404368001104672, "consumed_batches": 1431, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1981, "tokens": 149727798, "loss_tokens": 82392, "loss": 2.6176714668899894, "lr": 0.0002995481768026636, "optimizer_lrs": [0.01996987845351091, 0.0002995481768026636], "gradient_norm": 0.43614891171455383, "seconds": 0.2426387650000379, "nonpadding_tokens_per_second": 339566.5156801599, "ranks": [{"compute_seconds": 0.2394451160000699, "nonpadding_tokens": 9353, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24013480200005688, "nonpadding_tokens": 6108, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2373524009999528, "nonpadding_tokens": 1719, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23986344200000076, "nonpadding_tokens": 18708, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24012000599998373, "nonpadding_tokens": 5708, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23731170999997175, "nonpadding_tokens": 12788, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.239910387000009, "nonpadding_tokens": 8723, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2367025129999547, "nonpadding_tokens": 19285, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003901889999724517, "prefetch": {"data_wait_seconds": 0.013412915001097758, "consumed_batches": 1432, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1982, "tokens": 149786691, "loss_tokens": 58893, "loss": 2.73129844591038, "lr": 0.0002995477667071511, "optimizer_lrs": [0.019969851113810076, 0.0002995477667071511], "gradient_norm": 0.4200402498245239, "seconds": 0.32220129300003464, "nonpadding_tokens_per_second": 182783.2515867454, "ranks": [{"compute_seconds": 0.3192902220000633, "nonpadding_tokens": 5407, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3157524439999406, "nonpadding_tokens": 4266, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3164942959999735, "nonpadding_tokens": 18363, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3174431339999728, "nonpadding_tokens": 3830, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31949618800001645, "nonpadding_tokens": 8708, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3185244710000461, "nonpadding_tokens": 7539, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31579925699998057, "nonpadding_tokens": 4091, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31989725100004307, "nonpadding_tokens": 6689, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039413500007867697, "prefetch": {"data_wait_seconds": 0.013421927001061817, "consumed_batches": 1433, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1983, "tokens": 149840366, "loss_tokens": 53675, "loss": 2.4255573183511876, "lr": 0.0002995473927848982, "optimizer_lrs": [0.01996982618565988, 0.0002995473927848982], "gradient_norm": 0.3794202506542206, "seconds": 0.2786573750000798, "nonpadding_tokens_per_second": 192620.05895226935, "ranks": [{"compute_seconds": 0.276299538000103, "nonpadding_tokens": 4718, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2754615509999212, "nonpadding_tokens": 6202, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.274829999000076, "nonpadding_tokens": 11515, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27508282799999506, "nonpadding_tokens": 4137, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2755110829999694, "nonpadding_tokens": 12987, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27412417300001835, "nonpadding_tokens": 7588, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2715343730000086, "nonpadding_tokens": 2078, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2722693299999719, "nonpadding_tokens": 4450, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003793150000319656, "prefetch": {"data_wait_seconds": 0.01343192200097576, "consumed_batches": 1434, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1984, "tokens": 149886940, "loss_tokens": 46574, "loss": 2.4592124227036543, "lr": 0.0002995470682061486, "optimizer_lrs": [0.019969804547076576, 0.0002995470682061486], "gradient_norm": 0.4202004671096802, "seconds": 0.3161317950000466, "nonpadding_tokens_per_second": 147324.6308552834, "ranks": [{"compute_seconds": 0.3121911020000425, "nonpadding_tokens": 6485, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30967285200006245, "nonpadding_tokens": 9033, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31190991300002224, "nonpadding_tokens": 3873, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.312723869000024, "nonpadding_tokens": 3039, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3137430269999868, "nonpadding_tokens": 8205, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3134219819999089, "nonpadding_tokens": 9100, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31058340000004137, "nonpadding_tokens": 3156, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3114175120000482, "nonpadding_tokens": 3683, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035577400001329806, "prefetch": {"data_wait_seconds": 0.013440545000960924, "consumed_batches": 1435, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1985, "tokens": 149932084, "loss_tokens": 45144, "loss": 2.522712216739766, "lr": 0.0002995467534823575, "optimizer_lrs": [0.0199697835654905, 0.0002995467534823575], "gradient_norm": 0.46429064869880676, "seconds": 0.2428363999999874, "nonpadding_tokens_per_second": 185902.9371214626, "ranks": [{"compute_seconds": 0.19987771499995688, "nonpadding_tokens": 3899, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20787755599997126, "nonpadding_tokens": 3949, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20972699099991132, "nonpadding_tokens": 5898, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.20748103400001128, "nonpadding_tokens": 5253, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21152510100000654, "nonpadding_tokens": 4577, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20855568199999652, "nonpadding_tokens": 5092, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21111661700001605, "nonpadding_tokens": 10763, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2119814850000239, "nonpadding_tokens": 5713, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.012798713000051976, "prefetch": {"data_wait_seconds": 0.013449366001012208, "consumed_batches": 1436, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1986, "tokens": 149988879, "loss_tokens": 56795, "loss": 2.565385102121666, "lr": 0.00029954635737800065, "optimizer_lrs": [0.01996975715853338, 0.00029954635737800065], "gradient_norm": 0.5884736180305481, "seconds": 0.2601008560000082, "nonpadding_tokens_per_second": 218357.60509760954, "ranks": [{"compute_seconds": 0.25740880500006824, "nonpadding_tokens": 2919, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2539777970000614, "nonpadding_tokens": 7709, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2565095029999611, "nonpadding_tokens": 3735, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2552140809999628, "nonpadding_tokens": 4859, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2537096809999184, "nonpadding_tokens": 2635, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2562624949999872, "nonpadding_tokens": 17812, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2538592729999891, "nonpadding_tokens": 14211, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2578015689999802, "nonpadding_tokens": 2915, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000381774999937079, "prefetch": {"data_wait_seconds": 0.01345793300102116, "consumed_batches": 1437, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1987, "tokens": 150046942, "loss_tokens": 58063, "loss": 2.4250520447617245, "lr": 0.00029954595225171066, "optimizer_lrs": [0.019969730150114044, 0.00029954595225171066], "gradient_norm": 0.4706716239452362, "seconds": 0.24670221099995615, "nonpadding_tokens_per_second": 235356.62596882976, "ranks": [{"compute_seconds": 0.24267322000002878, "nonpadding_tokens": 10869, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24443110599997908, "nonpadding_tokens": 10504, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24322191699991436, "nonpadding_tokens": 3835, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24129772900005264, "nonpadding_tokens": 1601, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24427573200000552, "nonpadding_tokens": 13300, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24139207700000043, "nonpadding_tokens": 6797, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2399148399999831, "nonpadding_tokens": 7420, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24064907700005733, "nonpadding_tokens": 3737, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003567670000848011, "prefetch": {"data_wait_seconds": 0.013467066001069838, "consumed_batches": 1438, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1988, "tokens": 150121125, "loss_tokens": 74183, "loss": 2.3673938099025382, "lr": 0.00029954543438777826, "optimizer_lrs": [0.019969695625851887, 0.00029954543438777826], "gradient_norm": 0.4004063010215759, "seconds": 0.284687444000042, "nonpadding_tokens_per_second": 260576.9996656019, "ranks": [{"compute_seconds": 0.2804711749998887, "nonpadding_tokens": 12883, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2804240390000814, "nonpadding_tokens": 9894, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2809192819998998, "nonpadding_tokens": 15473, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27805762799994227, "nonpadding_tokens": 6595, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2822891890000392, "nonpadding_tokens": 1821, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2795894869999529, "nonpadding_tokens": 17199, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2764136260000214, "nonpadding_tokens": 3911, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28236801500008823, "nonpadding_tokens": 6407, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035376100004214095, "prefetch": {"data_wait_seconds": 0.01347687300103928, "consumed_batches": 1439, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1989, "tokens": 150205160, "loss_tokens": 84035, "loss": 2.6809586407449277, "lr": 0.00029954484739208614, "optimizer_lrs": [0.019969656492805743, 0.00029954484739208614], "gradient_norm": 0.4967813789844513, "seconds": 0.3126249330000519, "nonpadding_tokens_per_second": 268804.5358173209, "ranks": [{"compute_seconds": 0.30827100399994833, "nonpadding_tokens": 7991, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30647929099995963, "nonpadding_tokens": 12111, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3090476260000514, "nonpadding_tokens": 3318, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3097740859999476, "nonpadding_tokens": 12299, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3102239280000276, "nonpadding_tokens": 7404, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30648760799999764, "nonpadding_tokens": 13758, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3034920909999528, "nonpadding_tokens": 11899, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30426147699995454, "nonpadding_tokens": 15255, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034505499991155375, "prefetch": {"data_wait_seconds": 0.013486616001003426, "consumed_batches": 1440, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1990, "tokens": 150256171, "loss_tokens": 51011, "loss": 2.408543010331105, "lr": 0.00029954449088901537, "optimizer_lrs": [0.01996963272593436, 0.00029954449088901537], "gradient_norm": 0.4291881322860718, "seconds": 0.3007701230000066, "nonpadding_tokens_per_second": 169601.28715975847, "ranks": [{"compute_seconds": 0.29806738900003893, "nonpadding_tokens": 3857, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29625804699992386, "nonpadding_tokens": 3253, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2933912099999816, "nonpadding_tokens": 6781, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29394777499999236, "nonpadding_tokens": 5438, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.298203332000071, "nonpadding_tokens": 5728, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29523896800003513, "nonpadding_tokens": 7689, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2920432639999717, "nonpadding_tokens": 11169, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2945804080000016, "nonpadding_tokens": 7096, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004096890000937492, "prefetch": {"data_wait_seconds": 0.013496901001033734, "consumed_batches": 1441, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1991, "tokens": 150293206, "loss_tokens": 37035, "loss": 2.2998597188470367, "lr": 0.00029954423197340246, "optimizer_lrs": [0.0199696154648935, 0.00029954423197340246], "gradient_norm": 0.4320704936981201, "seconds": 0.2658270150000135, "nonpadding_tokens_per_second": 139319.92577954548, "ranks": [{"compute_seconds": 0.2629911899999797, "nonpadding_tokens": 8309, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26333172299996477, "nonpadding_tokens": 1488, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2626786970000694, "nonpadding_tokens": 5997, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26086387200007266, "nonpadding_tokens": 3716, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25943934200006424, "nonpadding_tokens": 6956, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2624132329999611, "nonpadding_tokens": 1807, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.263510434000068, "nonpadding_tokens": 4595, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26018179499999405, "nonpadding_tokens": 4167, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035851899997396686, "prefetch": {"data_wait_seconds": 0.013505700000976617, "consumed_batches": 1442, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1992, "tokens": 150341299, "loss_tokens": 48093, "loss": 2.4917667656935523, "lr": 0.00029954389564053373, "optimizer_lrs": [0.01996959304270225, 0.00029954389564053373], "gradient_norm": 0.39477118849754333, "seconds": 0.29606190499998775, "nonpadding_tokens_per_second": 162442.3783938092, "ranks": [{"compute_seconds": 0.23168284199994105, "nonpadding_tokens": 11194, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2400998119999258, "nonpadding_tokens": 2147, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24178654900003949, "nonpadding_tokens": 2428, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23789070700001957, "nonpadding_tokens": 6846, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24291427899993323, "nonpadding_tokens": 10161, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2404798219999975, "nonpadding_tokens": 5665, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23711166299995057, "nonpadding_tokens": 5235, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23730104000003394, "nonpadding_tokens": 4417, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.04512212399993132, "prefetch": {"data_wait_seconds": 0.013512808000953669, "consumed_batches": 1443, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1993, "tokens": 150380352, "loss_tokens": 39053, "loss": 2.1510479348577576, "lr": 0.0002995436224367539, "optimizer_lrs": [0.019969574829116925, 0.0002995436224367539], "gradient_norm": 0.4609588086605072, "seconds": 0.2560228539999798, "nonpadding_tokens_per_second": 152537.16373305907, "ranks": [{"compute_seconds": 0.2534627269999419, "nonpadding_tokens": 2692, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.252119265000033, "nonpadding_tokens": 3018, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25163755900007345, "nonpadding_tokens": 10650, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2521859849999828, "nonpadding_tokens": 3640, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2510123149999117, "nonpadding_tokens": 4178, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2536207570000215, "nonpadding_tokens": 9653, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2511381580000034, "nonpadding_tokens": 2128, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2508562119999169, "nonpadding_tokens": 3094, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004350860000386092, "prefetch": {"data_wait_seconds": 0.013520492000978379, "consumed_batches": 1444, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1994, "tokens": 150473659, "loss_tokens": 93307, "loss": 2.2244750527827493, "lr": 0.00029954296935665096, "optimizer_lrs": [0.0199695312904434, 0.00029954296935665096], "gradient_norm": 0.48586827516555786, "seconds": 0.3348468710000816, "nonpadding_tokens_per_second": 278655.7321599618, "ranks": [{"compute_seconds": 0.33048151600007714, "nonpadding_tokens": 13006, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3279607780000333, "nonpadding_tokens": 7775, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3310630600000195, "nonpadding_tokens": 11604, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3310295370000631, "nonpadding_tokens": 9672, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3274584399999867, "nonpadding_tokens": 10914, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32897433900006945, "nonpadding_tokens": 24587, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33123168199995234, "nonpadding_tokens": 11377, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3325046249999559, "nonpadding_tokens": 4372, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003423489999931917, "prefetch": {"data_wait_seconds": 0.013529136000968833, "consumed_batches": 1445, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1995, "tokens": 150532871, "loss_tokens": 59212, "loss": 2.515904187495778, "lr": 0.0002995425546746006, "optimizer_lrs": [0.019969503644973376, 0.0002995425546746006], "gradient_norm": 0.44683581590652466, "seconds": 0.2786946150000631, "nonpadding_tokens_per_second": 212461.94512939046, "ranks": [{"compute_seconds": 0.273462790999929, "nonpadding_tokens": 5879, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27609895000000506, "nonpadding_tokens": 7615, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2750299779999068, "nonpadding_tokens": 9452, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27632731799997146, "nonpadding_tokens": 7827, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27202365999994527, "nonpadding_tokens": 3614, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2748695720000569, "nonpadding_tokens": 18069, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2720033649999323, "nonpadding_tokens": 3864, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27610175100005563, "nonpadding_tokens": 2892, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037545800000771123, "prefetch": {"data_wait_seconds": 0.013536863000993549, "consumed_batches": 1446, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1996, "tokens": 150629484, "loss_tokens": 96613, "loss": 2.370724080869034, "lr": 0.00029954187765741724, "optimizer_lrs": [0.019969458510494482, 0.00029954187765741724], "gradient_norm": 0.3884090483188629, "seconds": 0.29048126800000773, "nonpadding_tokens_per_second": 332596.3173639046, "ranks": [{"compute_seconds": 0.2882556330000625, "nonpadding_tokens": 8062, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2837577910000846, "nonpadding_tokens": 10360, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28405579200000375, "nonpadding_tokens": 15049, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28498103100002936, "nonpadding_tokens": 17726, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28734769699997287, "nonpadding_tokens": 14507, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2860106399999722, "nonpadding_tokens": 16911, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2832508740000321, "nonpadding_tokens": 4329, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2840380099999038, "nonpadding_tokens": 9669, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037209699996765266, "prefetch": {"data_wait_seconds": 0.013546522000936534, "consumed_batches": 1447, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1997, "tokens": 150672211, "loss_tokens": 42727, "loss": 2.3341752214641796, "lr": 0.0002995415780878785, "optimizer_lrs": [0.019969438539191902, 0.0002995415780878785], "gradient_norm": 0.4446965754032135, "seconds": 0.2655118189999257, "nonpadding_tokens_per_second": 160923.15649425745, "ranks": [{"compute_seconds": 0.2604430940000384, "nonpadding_tokens": 2863, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2604201630000489, "nonpadding_tokens": 3488, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26163181300000815, "nonpadding_tokens": 10201, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26243029499994464, "nonpadding_tokens": 4161, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25610639399997126, "nonpadding_tokens": 6400, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25840577400003895, "nonpadding_tokens": 2698, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25583312599997043, "nonpadding_tokens": 3849, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2632022609999467, "nonpadding_tokens": 9067, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003469029999223494, "prefetch": {"data_wait_seconds": 0.013555062000889961, "consumed_batches": 1448, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1998, "tokens": 150728430, "loss_tokens": 56219, "loss": 2.349277490705989, "lr": 0.0002995411837736804, "optimizer_lrs": [0.019969412251578698, 0.0002995411837736804], "gradient_norm": 0.45705363154411316, "seconds": 0.26637785899993105, "nonpadding_tokens_per_second": 211049.8230260742, "ranks": [{"compute_seconds": 0.2626168179999695, "nonpadding_tokens": 3326, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26328520699996716, "nonpadding_tokens": 8295, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2607902470000454, "nonpadding_tokens": 10268, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26126210100005665, "nonpadding_tokens": 2910, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2639881879999848, "nonpadding_tokens": 6260, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2624080260000028, "nonpadding_tokens": 17250, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.263677303999998, "nonpadding_tokens": 4175, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26403628100001697, "nonpadding_tokens": 3735, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003736480000497977, "prefetch": {"data_wait_seconds": 0.013563381000906816, "consumed_batches": 1449, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 1999, "tokens": 150807167, "loss_tokens": 78737, "loss": 2.3367712685903705, "lr": 0.0002995406312361102, "optimizer_lrs": [0.019969375415740682, 0.0002995406312361102], "gradient_norm": 0.419599711894989, "seconds": 0.28165017000003445, "nonpadding_tokens_per_second": 279556.01802047685, "ranks": [{"compute_seconds": 0.2795021069999848, "nonpadding_tokens": 7658, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2782788289999871, "nonpadding_tokens": 10586, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27487141799997517, "nonpadding_tokens": 4627, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2756245890000173, "nonpadding_tokens": 11563, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2789799039999252, "nonpadding_tokens": 11070, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27657703500005937, "nonpadding_tokens": 17548, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27859932299998036, "nonpadding_tokens": 13444, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27469350500007295, "nonpadding_tokens": 2241, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034554000001207896, "prefetch": {"data_wait_seconds": 0.01357148900099503, "consumed_batches": 1450, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2000, "tokens": 150859926, "loss_tokens": 52759, "loss": 2.344670459068595, "lr": 0.00029954026081365046, "optimizer_lrs": [0.019969350720910033, 0.00029954026081365046], "gradient_norm": 0.4153258204460144, "seconds": 0.30813469999998233, "nonpadding_tokens_per_second": 171220.57334017567, "ranks": [{"compute_seconds": 0.30519921199993405, "nonpadding_tokens": 7818, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30381023299992194, "nonpadding_tokens": 7179, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3008761429999822, "nonpadding_tokens": 4280, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3016774830000486, "nonpadding_tokens": 6582, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2998083949999, "nonpadding_tokens": 9964, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30268727999998646, "nonpadding_tokens": 6354, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2997777350000206, "nonpadding_tokens": 7794, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3058086099999855, "nonpadding_tokens": 2788, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003604850001011073, "prefetch": {"data_wait_seconds": 0.013580373001062185, "consumed_batches": 1451, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2001, "tokens": 150933898, "loss_tokens": 73972, "loss": 2.554153936624669, "lr": 0.0002995397412031411, "optimizer_lrs": [0.01996931608020941, 0.0002995397412031411], "gradient_norm": 0.41468384861946106, "seconds": 0.2757586310000306, "nonpadding_tokens_per_second": 268249.0833804284, "ranks": [{"compute_seconds": 0.27229752499999904, "nonpadding_tokens": 7153, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27248053799996796, "nonpadding_tokens": 17115, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2693904750000229, "nonpadding_tokens": 21495, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.27219605000004776, "nonpadding_tokens": 2971, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27279081500000757, "nonpadding_tokens": 4923, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2696439220000002, "nonpadding_tokens": 9566, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2722095229999013, "nonpadding_tokens": 4849, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2691025350000018, "nonpadding_tokens": 5900, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047801400000935246, "prefetch": {"data_wait_seconds": 0.013589757001071234, "consumed_batches": 1452, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2002, "tokens": 150996296, "loss_tokens": 62398, "loss": 2.4019096665758517, "lr": 0.0002995393026654878, "optimizer_lrs": [0.019969286844365854, 0.0002995393026654878], "gradient_norm": 0.38978004455566406, "seconds": 0.24611198200000217, "nonpadding_tokens_per_second": 253534.99448880734, "ranks": [{"compute_seconds": 0.2201201510000601, "nonpadding_tokens": 6375, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2306997789999059, "nonpadding_tokens": 8437, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22788914399995974, "nonpadding_tokens": 7811, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22862789999999222, "nonpadding_tokens": 8729, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2314544760000672, "nonpadding_tokens": 18250, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22977746899994145, "nonpadding_tokens": 3667, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23073943200006397, "nonpadding_tokens": 3579, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2314960549999796, "nonpadding_tokens": 5550, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01120489599998109, "prefetch": {"data_wait_seconds": 0.013598947001085548, "consumed_batches": 1453, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2003, "tokens": 151043399, "loss_tokens": 47103, "loss": 2.5719293025391163, "lr": 0.0002995389714841211, "optimizer_lrs": [0.019969264765608074, 0.0002995389714841211], "gradient_norm": 0.4207114577293396, "seconds": 0.2298340710000275, "nonpadding_tokens_per_second": 204943.50465555806, "ranks": [{"compute_seconds": 0.22722714600001837, "nonpadding_tokens": 8441, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22314805999997134, "nonpadding_tokens": 13010, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21970729299994218, "nonpadding_tokens": 2810, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22474919700005103, "nonpadding_tokens": 3043, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21957616800000324, "nonpadding_tokens": 6146, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22204152700010127, "nonpadding_tokens": 3550, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22550385200008805, "nonpadding_tokens": 5576, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21971746400004122, "nonpadding_tokens": 4527, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003549450000264187, "prefetch": {"data_wait_seconds": 0.01360878700116075, "consumed_batches": 1454, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2004, "tokens": 151105924, "loss_tokens": 62525, "loss": 2.44047181127549, "lr": 0.0002995385316871162, "optimizer_lrs": [0.01996923544580775, 0.0002995385316871162], "gradient_norm": 1.071663737297058, "seconds": 0.34715136099998745, "nonpadding_tokens_per_second": 180108.756652699, "ranks": [{"compute_seconds": 0.34453298599999016, "nonpadding_tokens": 5297, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34069374299997435, "nonpadding_tokens": 6344, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3403167230000008, "nonpadding_tokens": 5494, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.34165268499998547, "nonpadding_tokens": 7500, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3443904669999256, "nonpadding_tokens": 10655, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34118455500004075, "nonpadding_tokens": 3202, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3398316220000197, "nonpadding_tokens": 12402, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34032163500000934, "nonpadding_tokens": 11631, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043834400003106566, "prefetch": {"data_wait_seconds": 0.013619069001222783, "consumed_batches": 1455, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2005, "tokens": 151174666, "loss_tokens": 68742, "loss": 2.180776490355241, "lr": 0.0002995380479185499, "optimizer_lrs": [0.019969203194569993, 0.0002995380479185499], "gradient_norm": 2.005065679550171, "seconds": 0.44955910299995594, "nonpadding_tokens_per_second": 152909.81662094546, "ranks": [{"compute_seconds": 0.4103431369999271, "nonpadding_tokens": 12348, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4137933050000129, "nonpadding_tokens": 12065, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.41045504999999594, "nonpadding_tokens": 12857, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.41328326600000764, "nonpadding_tokens": 3617, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.40932518700003584, "nonpadding_tokens": 3720, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4125010340000017, "nonpadding_tokens": 5799, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.40971659299998464, "nonpadding_tokens": 11162, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.41015213299999687, "nonpadding_tokens": 7174, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.020540194999966843, "prefetch": {"data_wait_seconds": 0.01362848200115252, "consumed_batches": 1456, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2006, "tokens": 151219533, "loss_tokens": 44867, "loss": 2.4017004424187043, "lr": 0.00029953773203270713, "optimizer_lrs": [0.01996918213551381, 0.00029953773203270713], "gradient_norm": 0.42342662811279297, "seconds": 0.2601627760000156, "nonpadding_tokens_per_second": 172457.4156604068, "ranks": [{"compute_seconds": 0.2577733629999557, "nonpadding_tokens": 8496, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2527974509999922, "nonpadding_tokens": 5981, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2516186010000183, "nonpadding_tokens": 2464, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25210249300005216, "nonpadding_tokens": 2806, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2502983660000382, "nonpadding_tokens": 8620, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25318744499998047, "nonpadding_tokens": 3966, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2561173809999673, "nonpadding_tokens": 4843, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25667658500003654, "nonpadding_tokens": 7691, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040796099995077384, "prefetch": {"data_wait_seconds": 0.013638310001127252, "consumed_batches": 1457, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2007, "tokens": 151254891, "loss_tokens": 35358, "loss": 2.5153784150687257, "lr": 0.0002995374830189847, "optimizer_lrs": [0.01996916553459898, 0.0002995374830189847], "gradient_norm": 0.4197545349597931, "seconds": 0.2557695689999946, "nonpadding_tokens_per_second": 138241.62169972906, "ranks": [{"compute_seconds": 0.2523398119999456, "nonpadding_tokens": 3419, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25189072700004544, "nonpadding_tokens": 1954, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24915827999996054, "nonpadding_tokens": 5075, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2526375489999282, "nonpadding_tokens": 3993, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2531461249999438, "nonpadding_tokens": 3723, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25045702199997777, "nonpadding_tokens": 7043, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2527097140000478, "nonpadding_tokens": 6268, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24870831500004442, "nonpadding_tokens": 3883, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000471379000032357, "prefetch": {"data_wait_seconds": 0.01364735200111511, "consumed_batches": 1458, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2008, "tokens": 151360379, "loss_tokens": 105488, "loss": 2.475278882432125, "lr": 0.0002995367397070634, "optimizer_lrs": [0.019969115980470895, 0.0002995367397070634], "gradient_norm": 0.36972877383232117, "seconds": 0.26732450500003324, "nonpadding_tokens_per_second": 394606.54757403134, "ranks": [{"compute_seconds": 0.26480175000006057, "nonpadding_tokens": 3255, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2643230199998925, "nonpadding_tokens": 11375, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2633964079999487, "nonpadding_tokens": 11978, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.263708174000044, "nonpadding_tokens": 21879, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26041196100004527, "nonpadding_tokens": 11545, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26196441199999754, "nonpadding_tokens": 20182, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2604435599999988, "nonpadding_tokens": 11800, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26049122400002034, "nonpadding_tokens": 13474, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004615809999677367, "prefetch": {"data_wait_seconds": 0.013656354001071236, "consumed_batches": 1459, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2009, "tokens": 151440789, "loss_tokens": 80410, "loss": 2.337106120196493, "lr": 0.00029953617270490207, "optimizer_lrs": [0.019969078180326807, 0.00029953617270490207], "gradient_norm": 0.6571254134178162, "seconds": 0.30486921800002165, "nonpadding_tokens_per_second": 263752.4395788436, "ranks": [{"compute_seconds": 0.30229192499996316, "nonpadding_tokens": 7637, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3000838760000306, "nonpadding_tokens": 10108, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30112097400001403, "nonpadding_tokens": 8213, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29723131100001865, "nonpadding_tokens": 6994, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2960831859999189, "nonpadding_tokens": 4559, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2987114150000707, "nonpadding_tokens": 10838, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2960087819999444, "nonpadding_tokens": 23140, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3025417669999797, "nonpadding_tokens": 8921, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003496250000125656, "prefetch": {"data_wait_seconds": 0.013665313001027357, "consumed_batches": 1460, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2010, "tokens": 151505993, "loss_tokens": 65204, "loss": 2.835353515505184, "lr": 0.0002995357126719846, "optimizer_lrs": [0.01996904751146564, 0.0002995357126719846], "gradient_norm": 1.12424635887146, "seconds": 0.25561886699995284, "nonpadding_tokens_per_second": 255082.89260984803, "ranks": [{"compute_seconds": 0.2532560079999939, "nonpadding_tokens": 11793, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2519915469999887, "nonpadding_tokens": 16329, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24922221199994965, "nonpadding_tokens": 2454, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2526171320000685, "nonpadding_tokens": 13266, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2532165880001003, "nonpadding_tokens": 5423, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2509544179999921, "nonpadding_tokens": 2346, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2483306489999677, "nonpadding_tokens": 1357, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2489646870000115, "nonpadding_tokens": 12236, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040433399999528774, "prefetch": {"data_wait_seconds": 0.01368229300096857, "consumed_batches": 1461, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2011, "tokens": 151562535, "loss_tokens": 56542, "loss": 2.6198555940716637, "lr": 0.00029953531356768253, "optimizer_lrs": [0.01996902090451217, 0.00029953531356768253], "gradient_norm": 0.6252116560935974, "seconds": 0.32935077600006935, "nonpadding_tokens_per_second": 171677.14218468426, "ranks": [{"compute_seconds": 0.32105604100001983, "nonpadding_tokens": 10188, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32452233300000444, "nonpadding_tokens": 3197, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32600914699992245, "nonpadding_tokens": 5522, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3268153540000185, "nonpadding_tokens": 2373, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3191508109999859, "nonpadding_tokens": 3143, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3219545179999841, "nonpadding_tokens": 5669, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32679820999999265, "nonpadding_tokens": 24080, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31947277800009033, "nonpadding_tokens": 2370, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003746410000076139, "prefetch": {"data_wait_seconds": 0.013690923000993394, "consumed_batches": 1462, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2012, "tokens": 151610899, "loss_tokens": 48364, "loss": 2.5404720582458027, "lr": 0.0002995349720523469, "optimizer_lrs": [0.019968998136823127, 0.0002995349720523469], "gradient_norm": 0.45656853914260864, "seconds": 0.3851297070000328, "nonpadding_tokens_per_second": 125578.47167057378, "ranks": [{"compute_seconds": 0.38247246299999915, "nonpadding_tokens": 9887, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37920948199996474, "nonpadding_tokens": 6904, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3763785219999818, "nonpadding_tokens": 1588, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3822965919999888, "nonpadding_tokens": 3240, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.38235455400001683, "nonpadding_tokens": 5409, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3801632980000704, "nonpadding_tokens": 1049, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3820669020000196, "nonpadding_tokens": 3086, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3827643920000128, "nonpadding_tokens": 17201, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038948500002788933, "prefetch": {"data_wait_seconds": 0.013701387000949694, "consumed_batches": 1463, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2013, "tokens": 151694631, "loss_tokens": 83732, "loss": 2.414127625937515, "lr": 0.00029953438049494275, "optimizer_lrs": [0.01996895869966285, 0.00029953438049494275], "gradient_norm": 0.4031001627445221, "seconds": 0.25906423200001427, "nonpadding_tokens_per_second": 323209.4193535578, "ranks": [{"compute_seconds": 0.2531320930000902, "nonpadding_tokens": 10701, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25326184499999727, "nonpadding_tokens": 10423, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25174026199999844, "nonpadding_tokens": 15695, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2559598529999221, "nonpadding_tokens": 4582, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25142573800007995, "nonpadding_tokens": 8701, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2539130059999479, "nonpadding_tokens": 16779, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.256603314999893, "nonpadding_tokens": 10345, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2518364539999993, "nonpadding_tokens": 6506, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043426499996712664, "prefetch": {"data_wait_seconds": 0.01371204800098269, "consumed_batches": 1464, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2014, "tokens": 151738417, "loss_tokens": 43786, "loss": 2.370132398198054, "lr": 0.000299534071002182, "optimizer_lrs": [0.019968938066812134, 0.000299534071002182], "gradient_norm": 0.3896105885505676, "seconds": 0.3221874510001044, "nonpadding_tokens_per_second": 135902.25151253893, "ranks": [{"compute_seconds": 0.3190519529999847, "nonpadding_tokens": 3714, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31859618599992245, "nonpadding_tokens": 2856, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3153991019999012, "nonpadding_tokens": 3256, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3160735139999815, "nonpadding_tokens": 5519, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31426928899998074, "nonpadding_tokens": 3371, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3172195920000149, "nonpadding_tokens": 3499, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3190503409999792, "nonpadding_tokens": 13834, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31977830700009235, "nonpadding_tokens": 7737, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003970390000631596, "prefetch": {"data_wait_seconds": 0.013720249000996887, "consumed_batches": 1465, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2015, "tokens": 151832222, "loss_tokens": 93805, "loss": 2.464277790362987, "lr": 0.00029953340761441265, "optimizer_lrs": [0.019968893840960844, 0.00029953340761441265], "gradient_norm": 0.5677450895309448, "seconds": 0.2906756170000335, "nonpadding_tokens_per_second": 322713.6867141807, "ranks": [{"compute_seconds": 0.2879693390000284, "nonpadding_tokens": 2224, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2843002319999641, "nonpadding_tokens": 6576, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2846411009999201, "nonpadding_tokens": 8785, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2882501049999746, "nonpadding_tokens": 15940, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2833202590001065, "nonpadding_tokens": 18244, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28586537000001044, "nonpadding_tokens": 21202, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2877522180000369, "nonpadding_tokens": 17401, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2842994659999931, "nonpadding_tokens": 3433, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037484900008166733, "prefetch": {"data_wait_seconds": 0.013729898001045626, "consumed_batches": 1466, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2016, "tokens": 151916567, "loss_tokens": 84345, "loss": 2.7268113847886655, "lr": 0.0002995328107254022, "optimizer_lrs": [0.019968854048360147, 0.0002995328107254022], "gradient_norm": 0.49499887228012085, "seconds": 0.38491144199997507, "nonpadding_tokens_per_second": 219128.32614626578, "ranks": [{"compute_seconds": 0.3821660180000208, "nonpadding_tokens": 14479, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3815733649998947, "nonpadding_tokens": 7837, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3786731980000013, "nonpadding_tokens": 12564, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3795046849999153, "nonpadding_tokens": 15304, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3779244810000364, "nonpadding_tokens": 8037, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38065523899990694, "nonpadding_tokens": 16556, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3814650880000272, "nonpadding_tokens": 4511, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3825191170000153, "nonpadding_tokens": 5057, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038225100001909595, "prefetch": {"data_wait_seconds": 0.013739732001070593, "consumed_batches": 1467, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2017, "tokens": 151969791, "loss_tokens": 53224, "loss": 2.5870797830865775, "lr": 0.00029953243387611936, "optimizer_lrs": [0.019968828925074626, 0.00029953243387611936], "gradient_norm": 0.4116683900356293, "seconds": 0.34052151999992475, "nonpadding_tokens_per_second": 156301.42846775663, "ranks": [{"compute_seconds": 0.3367007690000037, "nonpadding_tokens": 6083, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33482136800000717, "nonpadding_tokens": 11922, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3371326760000102, "nonpadding_tokens": 5086, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3381475930000306, "nonpadding_tokens": 8727, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3380533629999718, "nonpadding_tokens": 4957, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3364920220000158, "nonpadding_tokens": 12097, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33407113300006586, "nonpadding_tokens": 1842, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3349287409999988, "nonpadding_tokens": 2510, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004165120000152456, "prefetch": {"data_wait_seconds": 0.013748848001071678, "consumed_batches": 1468, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2018, "tokens": 152065901, "loss_tokens": 96110, "loss": 2.5118682369160337, "lr": 0.000299531752990925, "optimizer_lrs": [0.019968783532728334, 0.000299531752990925], "gradient_norm": 0.40219634771347046, "seconds": 0.3155795779999835, "nonpadding_tokens_per_second": 304550.7589848068, "ranks": [{"compute_seconds": 0.3113936880000665, "nonpadding_tokens": 24458, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3127697510000189, "nonpadding_tokens": 5848, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31211162900001455, "nonpadding_tokens": 7025, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3126466599999276, "nonpadding_tokens": 18896, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30914679800002887, "nonpadding_tokens": 28413, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3103400039999542, "nonpadding_tokens": 3630, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30895321800005604, "nonpadding_tokens": 2139, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3132571809999263, "nonpadding_tokens": 5701, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038270400000328664, "prefetch": {"data_wait_seconds": 0.013758751001049063, "consumed_batches": 1469, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2019, "tokens": 152125792, "loss_tokens": 59891, "loss": 2.4648744698702645, "lr": 0.0002995313284468305, "optimizer_lrs": [0.019968755229788703, 0.0002995313284468305], "gradient_norm": 0.45007777214050293, "seconds": 0.2335110789999817, "nonpadding_tokens_per_second": 256480.335992986, "ranks": [{"compute_seconds": 0.22927292500003205, "nonpadding_tokens": 7255, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23061193099999855, "nonpadding_tokens": 4101, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22982423799999196, "nonpadding_tokens": 4763, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23056824599996162, "nonpadding_tokens": 11991, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23094577100005154, "nonpadding_tokens": 10086, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22969806300000073, "nonpadding_tokens": 3341, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23083809800004929, "nonpadding_tokens": 8578, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23115628199991534, "nonpadding_tokens": 9776, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003876490000038757, "prefetch": {"data_wait_seconds": 0.01376690500099187, "consumed_batches": 1470, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2020, "tokens": 152182151, "loss_tokens": 56359, "loss": 2.586201571621214, "lr": 0.0002995309287643313, "optimizer_lrs": [0.019968728584288754, 0.0002995309287643313], "gradient_norm": 0.4197356104850769, "seconds": 0.22888384700002007, "nonpadding_tokens_per_second": 246234.06473937436, "ranks": [{"compute_seconds": 0.22472783699993215, "nonpadding_tokens": 4106, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22628678799992485, "nonpadding_tokens": 7758, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22353930500003116, "nonpadding_tokens": 13717, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22417676899999606, "nonpadding_tokens": 8376, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2263459440000588, "nonpadding_tokens": 13111, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22391840299997057, "nonpadding_tokens": 2385, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22611427399999684, "nonpadding_tokens": 2317, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2232152879998921, "nonpadding_tokens": 4589, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004825649999702364, "prefetch": {"data_wait_seconds": 0.013775143001069523, "consumed_batches": 1471, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2021, "tokens": 152258948, "loss_tokens": 76797, "loss": 2.7556872745680168, "lr": 0.0002995303838674275, "optimizer_lrs": [0.0199686922578285, 0.0002995303838674275], "gradient_norm": 0.38531193137168884, "seconds": 0.27251190000004044, "nonpadding_tokens_per_second": 281811.5465782911, "ranks": [{"compute_seconds": 0.26780467400010366, "nonpadding_tokens": 2831, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26688635200002864, "nonpadding_tokens": 7214, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26940881400003036, "nonpadding_tokens": 4491, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2678335909999987, "nonpadding_tokens": 11689, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26665324000009605, "nonpadding_tokens": 3434, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.267992247000052, "nonpadding_tokens": 16519, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27019043599989345, "nonpadding_tokens": 22728, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26690578000000187, "nonpadding_tokens": 7891, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003986929999655331, "prefetch": {"data_wait_seconds": 0.013784573001046851, "consumed_batches": 1472, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2022, "tokens": 152296645, "loss_tokens": 37697, "loss": 2.421084983553068, "lr": 0.0002995301162807837, "optimizer_lrs": [0.019968674418718916, 0.0002995301162807837], "gradient_norm": 0.4620601534843445, "seconds": 0.32612430000006043, "nonpadding_tokens_per_second": 115590.89586391758, "ranks": [{"compute_seconds": 0.31596716700005345, "nonpadding_tokens": 5666, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31973249999998643, "nonpadding_tokens": 7049, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31683337999993455, "nonpadding_tokens": 5522, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3191728570000123, "nonpadding_tokens": 2722, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3151152489999731, "nonpadding_tokens": 4595, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31819496200000685, "nonpadding_tokens": 5786, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3197351409999101, "nonpadding_tokens": 1116, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3205201869999428, "nonpadding_tokens": 5241, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003505760000734881, "prefetch": {"data_wait_seconds": 0.013792353001079505, "consumed_batches": 1473, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2023, "tokens": 152331701, "loss_tokens": 35056, "loss": 2.501322660528868, "lr": 0.0002995298673726202, "optimizer_lrs": [0.01996865782484135, 0.0002995298673726202], "gradient_norm": 0.48252734541893005, "seconds": 0.17605084399997395, "nonpadding_tokens_per_second": 199124.2938886745, "ranks": [{"compute_seconds": 0.17282165100004931, "nonpadding_tokens": 1574, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.17012192999993658, "nonpadding_tokens": 9508, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.169766303000074, "nonpadding_tokens": 3915, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.17103399400002672, "nonpadding_tokens": 3468, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.17363822300001175, "nonpadding_tokens": 4721, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1703619909999361, "nonpadding_tokens": 3285, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.1685165380000626, "nonpadding_tokens": 4183, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.17362383500005762, "nonpadding_tokens": 4402, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003844110000272849, "prefetch": {"data_wait_seconds": 0.013800713001117515, "consumed_batches": 1474, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2024, "tokens": 152382468, "loss_tokens": 50767, "loss": 2.495078465095436, "lr": 0.0002995295067949912, "optimizer_lrs": [0.01996863378633275, 0.0002995295067949912], "gradient_norm": 0.448810338973999, "seconds": 0.23980241399999613, "nonpadding_tokens_per_second": 211703.45683009186, "ranks": [{"compute_seconds": 0.23688317400001324, "nonpadding_tokens": 4192, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23576195299995106, "nonpadding_tokens": 7077, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2335194480000382, "nonpadding_tokens": 1376, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.23743624999997337, "nonpadding_tokens": 7948, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23250573899997562, "nonpadding_tokens": 6444, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23487465300001986, "nonpadding_tokens": 3538, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2322898829999076, "nonpadding_tokens": 8979, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2331322850000106, "nonpadding_tokens": 11213, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037584100005005894, "prefetch": {"data_wait_seconds": 0.013810430001129248, "consumed_batches": 1475, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2025, "tokens": 152446662, "loss_tokens": 64194, "loss": 2.5915423365112003, "lr": 0.0002995290506532209, "optimizer_lrs": [0.019968603376881396, 0.0002995290506532209], "gradient_norm": 0.5000394582748413, "seconds": 0.28874884099991505, "nonpadding_tokens_per_second": 222317.77546777716, "ranks": [{"compute_seconds": 0.2860259500000666, "nonpadding_tokens": 7674, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2824366769999642, "nonpadding_tokens": 14572, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2862286659999427, "nonpadding_tokens": 10276, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28628709399993113, "nonpadding_tokens": 4527, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28064568000002055, "nonpadding_tokens": 5988, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28384350900000754, "nonpadding_tokens": 4203, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28111045200000717, "nonpadding_tokens": 6216, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2815011800000775, "nonpadding_tokens": 10738, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037838499997633335, "prefetch": {"data_wait_seconds": 0.013820455001180676, "consumed_batches": 1476, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2026, "tokens": 152595380, "loss_tokens": 148718, "loss": 4.6341998782931455, "lr": 0.00029952799306368706, "optimizer_lrs": [0.01996853287091247, 0.00029952799306368706], "gradient_norm": 2.4281458854675293, "seconds": 0.2643258580000065, "nonpadding_tokens_per_second": 562631.295800036, "ranks": [{"compute_seconds": 0.23056468299989774, "nonpadding_tokens": 3643, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23849699599998075, "nonpadding_tokens": 2742, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24157105500000853, "nonpadding_tokens": 10393, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.24169242700008908, "nonpadding_tokens": 25498, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23771819199998845, "nonpadding_tokens": 21553, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24133730099993045, "nonpadding_tokens": 27446, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23814279099997293, "nonpadding_tokens": 25762, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24266565600009926, "nonpadding_tokens": 31681, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.008916973999930633, "prefetch": {"data_wait_seconds": 0.013828865001144663, "consumed_batches": 1477, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2027, "tokens": 152717763, "loss_tokens": 122383, "loss": 5.005173308384334, "lr": 0.0002995271218640964, "optimizer_lrs": [0.01996847479093976, 0.0002995271218640964], "gradient_norm": 2.582008123397827, "seconds": 0.3999916950000397, "nonpadding_tokens_per_second": 305963.8525744587, "ranks": [{"compute_seconds": 0.3952230290000216, "nonpadding_tokens": 26208, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.39370006399997237, "nonpadding_tokens": 19738, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.394517106999956, "nonpadding_tokens": 18033, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.3969926020000685, "nonpadding_tokens": 12501, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3977130099999613, "nonpadding_tokens": 15727, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39458569600003557, "nonpadding_tokens": 13370, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3974943999999141, "nonpadding_tokens": 13892, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.393953085000021, "nonpadding_tokens": 2914, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036140499992143305, "prefetch": {"data_wait_seconds": 0.013838571001045352, "consumed_batches": 1478, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2028, "tokens": 152775367, "loss_tokens": 57604, "loss": 2.5226386730956185, "lr": 0.00029952671152486154, "optimizer_lrs": [0.01996844743499077, 0.00029952671152486154], "gradient_norm": 0.6687823534011841, "seconds": 0.2952155269999821, "nonpadding_tokens_per_second": 195125.23811121727, "ranks": [{"compute_seconds": 0.2917112320000115, "nonpadding_tokens": 1689, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28933020699992085, "nonpadding_tokens": 5053, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29173516699995616, "nonpadding_tokens": 16628, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.29193733099998553, "nonpadding_tokens": 18274, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2928733610000336, "nonpadding_tokens": 5614, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29004459300006147, "nonpadding_tokens": 4521, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28887579900003857, "nonpadding_tokens": 2300, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2890978879999011, "nonpadding_tokens": 3525, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004021139999395018, "prefetch": {"data_wait_seconds": 0.013861342001021626, "consumed_batches": 1479, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2029, "tokens": 152842529, "loss_tokens": 67162, "loss": 2.6362407778952384, "lr": 0.0002995262328753828, "optimizer_lrs": [0.019968415525025523, 0.0002995262328753828], "gradient_norm": 0.49097883701324463, "seconds": 0.3086994320000258, "nonpadding_tokens_per_second": 217564.37828494087, "ranks": [{"compute_seconds": 0.30590701699998135, "nonpadding_tokens": 9362, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30207233999999517, "nonpadding_tokens": 3437, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30251961299995855, "nonpadding_tokens": 10442, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30308085000001483, "nonpadding_tokens": 9608, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.301608242000043, "nonpadding_tokens": 11156, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3047195740000461, "nonpadding_tokens": 16073, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3013091530000338, "nonpadding_tokens": 3116, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30618415200001436, "nonpadding_tokens": 3968, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034046700000089913, "prefetch": {"data_wait_seconds": 0.01386881300102516, "consumed_batches": 1480, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2030, "tokens": 152894823, "loss_tokens": 52294, "loss": 2.393788902646575, "lr": 0.0002995258600198037, "optimizer_lrs": [0.019968390667986915, 0.0002995258600198037], "gradient_norm": 0.505229115486145, "seconds": 0.3201669520000223, "nonpadding_tokens_per_second": 163333.53481153905, "ranks": [{"compute_seconds": 0.2769801390001021, "nonpadding_tokens": 7467, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3074462139999241, "nonpadding_tokens": 4902, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3073212040000044, "nonpadding_tokens": 4980, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.30797124800005804, "nonpadding_tokens": 6411, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3084820420000369, "nonpadding_tokens": 11788, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3062132570000813, "nonpadding_tokens": 5961, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3030457180000212, "nonpadding_tokens": 8456, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3091328699999849, "nonpadding_tokens": 2329, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004422419999627891, "prefetch": {"data_wait_seconds": 0.013879557000976206, "consumed_batches": 1481, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2031, "tokens": 152963534, "loss_tokens": 68711, "loss": 2.641585772292646, "lr": 0.0002995253698886562, "optimizer_lrs": [0.019968357992577082, 0.0002995253698886562], "gradient_norm": 0.4789961576461792, "seconds": 0.27002229899994745, "nonpadding_tokens_per_second": 254464.16927223248, "ranks": [{"compute_seconds": 0.26602540399994723, "nonpadding_tokens": 5829, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26442575299995497, "nonpadding_tokens": 17451, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2669711240000652, "nonpadding_tokens": 2152, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26771220699993137, "nonpadding_tokens": 3765, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2641193059999978, "nonpadding_tokens": 15389, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26540778300000056, "nonpadding_tokens": 4900, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2675449490000119, "nonpadding_tokens": 9981, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2647228849999692, "nonpadding_tokens": 9244, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004049140000006446, "prefetch": {"data_wait_seconds": 0.013888906001014334, "consumed_batches": 1482, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2032, "tokens": 153055493, "loss_tokens": 91959, "loss": 2.569595553453169, "lr": 0.0002995247135287539, "optimizer_lrs": [0.01996831423525026, 0.0002995247135287539], "gradient_norm": 0.4696281850337982, "seconds": 0.2896851369999922, "nonpadding_tokens_per_second": 317444.6606144052, "ranks": [{"compute_seconds": 0.2859893090000014, "nonpadding_tokens": 11066, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28620306100003745, "nonpadding_tokens": 11743, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2832804209999722, "nonpadding_tokens": 14114, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2842317559999401, "nonpadding_tokens": 16502, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28243569400001434, "nonpadding_tokens": 9392, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.283594694000044, "nonpadding_tokens": 4523, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28593586899989987, "nonpadding_tokens": 10830, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28737273499996263, "nonpadding_tokens": 13789, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003353039999183238, "prefetch": {"data_wait_seconds": 0.013897233000946585, "consumed_batches": 1483, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2033, "tokens": 153134051, "loss_tokens": 78558, "loss": 2.6337200698846712, "lr": 0.00029952415246029137, "optimizer_lrs": [0.019968276830686093, 0.00029952415246029137], "gradient_norm": 0.40727698802948, "seconds": 0.367797386999996, "nonpadding_tokens_per_second": 213590.42444747128, "ranks": [{"compute_seconds": 0.3655305340000723, "nonpadding_tokens": 20788, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3604105960000652, "nonpadding_tokens": 10694, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3611134590000802, "nonpadding_tokens": 11233, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.36369746200000463, "nonpadding_tokens": 12527, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36008297399996536, "nonpadding_tokens": 2512, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36284548700007235, "nonpadding_tokens": 7032, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35968919799995547, "nonpadding_tokens": 9533, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36051745199995366, "nonpadding_tokens": 4239, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038192200008779764, "prefetch": {"data_wait_seconds": 0.013906357000976755, "consumed_batches": 1484, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2034, "tokens": 153239406, "loss_tokens": 105355, "loss": 2.3258836196668407, "lr": 0.00029952339948653467, "optimizer_lrs": [0.019968226632435648, 0.00029952339948653467], "gradient_norm": 0.4899233877658844, "seconds": 0.26818885800003045, "nonpadding_tokens_per_second": 392838.8404561835, "ranks": [{"compute_seconds": 0.2630643559999726, "nonpadding_tokens": 20963, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26482274200009215, "nonpadding_tokens": 8918, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26119754100000137, "nonpadding_tokens": 12964, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.262280517000022, "nonpadding_tokens": 17361, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2604329250000319, "nonpadding_tokens": 14858, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26318028999992293, "nonpadding_tokens": 2965, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2606166049999956, "nonpadding_tokens": 12139, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26571168099997067, "nonpadding_tokens": 15187, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004085350000195831, "prefetch": {"data_wait_seconds": 0.013915509000938187, "consumed_batches": 1485, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2035, "tokens": 153293364, "loss_tokens": 53958, "loss": 2.4168018991623113, "lr": 0.0002995230136178039, "optimizer_lrs": [0.019968200907853593, 0.0002995230136178039], "gradient_norm": 0.42532479763031006, "seconds": 0.23310227899992242, "nonpadding_tokens_per_second": 231477.7883403618, "ranks": [{"compute_seconds": 0.23076622400003544, "nonpadding_tokens": 3394, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.227395032000004, "nonpadding_tokens": 2264, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22752406699999028, "nonpadding_tokens": 9873, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22819355900003302, "nonpadding_tokens": 5045, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23078309799996077, "nonpadding_tokens": 11155, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2294515069999079, "nonpadding_tokens": 5695, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2262987780000003, "nonpadding_tokens": 10943, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23079593700003898, "nonpadding_tokens": 5589, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033877700002449274, "prefetch": {"data_wait_seconds": 0.013924219000912785, "consumed_batches": 1486, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2036, "tokens": 153351700, "loss_tokens": 58336, "loss": 2.2709810944356827, "lr": 0.00029952259626542406, "optimizer_lrs": [0.019968173084361607, 0.00029952259626542406], "gradient_norm": 0.4291146397590637, "seconds": 0.24077177599997412, "nonpadding_tokens_per_second": 242287.53456553924, "ranks": [{"compute_seconds": 0.21292292700002236, "nonpadding_tokens": 12498, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2154927230000112, "nonpadding_tokens": 12516, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2125324300000102, "nonpadding_tokens": 3087, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2132899030000317, "nonpadding_tokens": 9953, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21126697099998637, "nonpadding_tokens": 12948, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21421136100002514, "nonpadding_tokens": 2180, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21509321200005616, "nonpadding_tokens": 2851, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21553421500004788, "nonpadding_tokens": 2303, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.016782178999960706, "prefetch": {"data_wait_seconds": 0.013933749000855755, "consumed_batches": 1487, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2037, "tokens": 153393264, "loss_tokens": 41564, "loss": 2.620301861286209, "lr": 0.0002995222987935704, "optimizer_lrs": [0.019968153252904693, 0.0002995222987935704], "gradient_norm": 0.4732920527458191, "seconds": 0.2685198469999932, "nonpadding_tokens_per_second": 154789.30315345014, "ranks": [{"compute_seconds": 0.2650558309999269, "nonpadding_tokens": 4821, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26573542000005546, "nonpadding_tokens": 2343, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2625462910000351, "nonpadding_tokens": 3552, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2634676480000735, "nonpadding_tokens": 3899, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2618698869999889, "nonpadding_tokens": 2716, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26428500399993027, "nonpadding_tokens": 11551, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2620041670001001, "nonpadding_tokens": 9944, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2662392479999198, "nonpadding_tokens": 2738, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036870700000690704, "prefetch": {"data_wait_seconds": 0.013941646000787955, "consumed_batches": 1488, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2038, "tokens": 153434272, "loss_tokens": 41008, "loss": 2.913308074034335, "lr": 0.00029952200521034874, "optimizer_lrs": [0.019968133680689917, 0.00029952200521034874], "gradient_norm": 0.47147250175476074, "seconds": 0.2715925930000367, "nonpadding_tokens_per_second": 150990.8629945385, "ranks": [{"compute_seconds": 0.26896183399992424, "nonpadding_tokens": 11747, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26932835999991767, "nonpadding_tokens": 4723, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26559762300007606, "nonpadding_tokens": 3498, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.26656315599996105, "nonpadding_tokens": 8931, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2653212679999797, "nonpadding_tokens": 4821, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.268333814000016, "nonpadding_tokens": 1318, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2651970680000204, "nonpadding_tokens": 1921, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26633670900002926, "nonpadding_tokens": 4049, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038970100001733954, "prefetch": {"data_wait_seconds": 0.013951016000831373, "consumed_batches": 1489, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2039, "tokens": 153480738, "loss_tokens": 46466, "loss": 2.5182519207592646, "lr": 0.00029952167244358266, "optimizer_lrs": [0.019968111496238845, 0.00029952167244358266], "gradient_norm": 0.5666449069976807, "seconds": 0.26533768499996313, "nonpadding_tokens_per_second": 175120.24347392062, "ranks": [{"compute_seconds": 0.2626197490000095, "nonpadding_tokens": 6413, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2590446489999749, "nonpadding_tokens": 12623, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25951692900002854, "nonpadding_tokens": 3422, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.25946514800000386, "nonpadding_tokens": 2324, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25833449800006747, "nonpadding_tokens": 3970, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2612086440000212, "nonpadding_tokens": 2125, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2622951739999735, "nonpadding_tokens": 9373, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26302345700003116, "nonpadding_tokens": 6216, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003567829999155947, "prefetch": {"data_wait_seconds": 0.013958889000903696, "consumed_batches": 1490, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2040, "tokens": 153522034, "loss_tokens": 41296, "loss": 2.189752034095312, "lr": 0.0002995213766048178, "optimizer_lrs": [0.01996809177365452, 0.0002995213766048178], "gradient_norm": 0.5165738463401794, "seconds": 0.29147280400002273, "nonpadding_tokens_per_second": 141680.4567468215, "ranks": [{"compute_seconds": 0.28659217400002035, "nonpadding_tokens": 5904, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28908754999997655, "nonpadding_tokens": 4002, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28587828199999876, "nonpadding_tokens": 2823, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.28618590100006713, "nonpadding_tokens": 2238, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2845297149999624, "nonpadding_tokens": 11577, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2862893390000636, "nonpadding_tokens": 9012, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.288710748000085, "nonpadding_tokens": 3122, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28587349899999026, "nonpadding_tokens": 2618, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042941200001678226, "prefetch": {"data_wait_seconds": 0.013968935000960414, "consumed_batches": 1491, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2041, "tokens": 153577709, "loss_tokens": 55675, "loss": 2.3130433318365515, "lr": 0.00029952097761239697, "optimizer_lrs": [0.019968065174159798, 0.00029952097761239697], "gradient_norm": 0.5882561206817627, "seconds": 0.28474309499995343, "nonpadding_tokens_per_second": 195527.12946387377, "ranks": [{"compute_seconds": 0.2800562820000323, "nonpadding_tokens": 2481, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28180166499998904, "nonpadding_tokens": 5203, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28067940000005365, "nonpadding_tokens": 2500, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.2792051179999362, "nonpadding_tokens": 4403, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28207418800002415, "nonpadding_tokens": 13725, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28046770399998877, "nonpadding_tokens": 16849, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2814779110000245, "nonpadding_tokens": 5868, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28240838299996085, "nonpadding_tokens": 4646, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003904540000121415, "prefetch": {"data_wait_seconds": 0.013987048000970503, "consumed_batches": 1492, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2042, "tokens": 153625826, "loss_tokens": 48117, "loss": 2.4950613543550095, "lr": 0.00029952063265036916, "optimizer_lrs": [0.019968042176691278, 0.00029952063265036916], "gradient_norm": 0.4694799482822418, "seconds": 0.268362758999956, "nonpadding_tokens_per_second": 179298.35040937216, "ranks": [{"compute_seconds": 0.26453416799995466, "nonpadding_tokens": 6216, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2618436139999858, "nonpadding_tokens": 3907, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2620199570000068, "nonpadding_tokens": 3062, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.262708994000036, "nonpadding_tokens": 3910, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2609884979999606, "nonpadding_tokens": 7350, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2634261019999258, "nonpadding_tokens": 9453, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2654931329999499, "nonpadding_tokens": 12279, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.266025720000016, "nonpadding_tokens": 1940, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003876949999721546, "prefetch": {"data_wait_seconds": 0.013994948000984664, "consumed_batches": 1493, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2043, "tokens": 153687619, "loss_tokens": 61793, "loss": 2.3879191413266874, "lr": 0.0002995201894601058, "optimizer_lrs": [0.019968012630673722, 0.0002995201894601058], "gradient_norm": 0.4585719406604767, "seconds": 0.27059576800002105, "nonpadding_tokens_per_second": 228359.07766301502, "ranks": [{"compute_seconds": 0.2290369099999907, "nonpadding_tokens": 11903, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22550381700000344, "nonpadding_tokens": 4888, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22501736399999572, "nonpadding_tokens": 8677, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18360565760}, {"compute_seconds": 0.22814421800001128, "nonpadding_tokens": 2669, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22385812400000304, "nonpadding_tokens": 3025, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22635889399998632, "nonpadding_tokens": 8279, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22368865400005689, "nonpadding_tokens": 1635, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22907389500005593, "nonpadding_tokens": 20717, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.010120669000002636, "prefetch": {"data_wait_seconds": 0.014003537000988331, "consumed_batches": 1494, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2044, "tokens": 153754385, "loss_tokens": 66766, "loss": 2.4410667948506726, "lr": 0.0002995197103728615, "optimizer_lrs": [0.0199679806915241, 0.0002995197103728615], "gradient_norm": 0.42184147238731384, "seconds": 0.2770789019999711, "nonpadding_tokens_per_second": 240963.8536823961, "ranks": [{"compute_seconds": 0.2718225099999927, "nonpadding_tokens": 6375, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27072432500006016, "nonpadding_tokens": 30033, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27110866599991823, "nonpadding_tokens": 4847, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2739091419999795, "nonpadding_tokens": 4904, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27081353099993066, "nonpadding_tokens": 2069, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27163064000001214, "nonpadding_tokens": 3313, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27035437300003196, "nonpadding_tokens": 6133, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27469775700001264, "nonpadding_tokens": 9092, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040268400005061267, "prefetch": {"data_wait_seconds": 0.014010859000975984, "consumed_batches": 1495, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2045, "tokens": 153823214, "loss_tokens": 68829, "loss": 2.653263431838324, "lr": 0.00029951921623250796, "optimizer_lrs": [0.019967947748833866, 0.00029951921623250796], "gradient_norm": 0.38141414523124695, "seconds": 0.36060204199998225, "nonpadding_tokens_per_second": 190872.46322360923, "ranks": [{"compute_seconds": 0.35547890899999857, "nonpadding_tokens": 5505, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35642984499997965, "nonpadding_tokens": 3647, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.353522607000059, "nonpadding_tokens": 10652, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35714460600001985, "nonpadding_tokens": 4068, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35306138800001463, "nonpadding_tokens": 2624, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3551264049999645, "nonpadding_tokens": 3108, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3528078499999765, "nonpadding_tokens": 18019, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3583175090000168, "nonpadding_tokens": 21206, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034188500001164357, "prefetch": {"data_wait_seconds": 0.014018618001003347, "consumed_batches": 1496, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2046, "tokens": 153903465, "loss_tokens": 80251, "loss": 2.257555591207586, "lr": 0.0002995186397705692, "optimizer_lrs": [0.019967909318037946, 0.0002995186397705692], "gradient_norm": 0.3922078013420105, "seconds": 0.33970884699999715, "nonpadding_tokens_per_second": 236234.64831341492, "ranks": [{"compute_seconds": 0.3337541019999435, "nonpadding_tokens": 24797, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3357433639999954, "nonpadding_tokens": 7479, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3359172849999368, "nonpadding_tokens": 11461, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3362547109999241, "nonpadding_tokens": 8157, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33172603999992134, "nonpadding_tokens": 3107, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33476501699999517, "nonpadding_tokens": 3231, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33650282500002504, "nonpadding_tokens": 6917, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33736346900002445, "nonpadding_tokens": 15102, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034458799996173184, "prefetch": {"data_wait_seconds": 0.014026973000980547, "consumed_batches": 1497, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2047, "tokens": 153952827, "loss_tokens": 49362, "loss": 2.569874093432195, "lr": 0.0002995182850203811, "optimizer_lrs": [0.019967885668025407, 0.0002995182850203811], "gradient_norm": 0.453252911567688, "seconds": 0.2658722259999422, "nonpadding_tokens_per_second": 185660.6112742695, "ranks": [{"compute_seconds": 0.26153546199998345, "nonpadding_tokens": 3576, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2597264539999742, "nonpadding_tokens": 23287, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2620961449999868, "nonpadding_tokens": 2845, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2609720350000089, "nonpadding_tokens": 2782, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26360993499997676, "nonpadding_tokens": 5755, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26211392699997305, "nonpadding_tokens": 5470, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26276504000009027, "nonpadding_tokens": 3116, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25990162400000827, "nonpadding_tokens": 2531, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034723499993560836, "prefetch": {"data_wait_seconds": 0.014035412000978909, "consumed_batches": 1498, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2048, "tokens": 154005263, "loss_tokens": 52436, "loss": 2.4489136161225113, "lr": 0.0002995179080353817, "optimizer_lrs": [0.019967860535692116, 0.0002995179080353817], "gradient_norm": 0.44062086939811707, "seconds": 0.24437513899999885, "nonpadding_tokens_per_second": 214571.74495970414, "ranks": [{"compute_seconds": 0.19867274800003543, "nonpadding_tokens": 8824, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.207681920999903, "nonpadding_tokens": 5413, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20472137299998394, "nonpadding_tokens": 2481, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20826900899999146, "nonpadding_tokens": 4336, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20386946899998293, "nonpadding_tokens": 14146, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20602918700001283, "nonpadding_tokens": 7179, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20315741000001708, "nonpadding_tokens": 5122, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20351381300008597, "nonpadding_tokens": 4935, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.02473351300000104, "prefetch": {"data_wait_seconds": 0.01404395800098257, "consumed_batches": 1499, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2049, "tokens": 154106989, "loss_tokens": 101726, "loss": 2.3011955829876336, "lr": 0.0002995171762635735, "optimizer_lrs": [0.019967811750904903, 0.0002995171762635735], "gradient_norm": 0.5050848126411438, "seconds": 0.3247596620000195, "nonpadding_tokens_per_second": 313234.7144763129, "ranks": [{"compute_seconds": 0.32061264000003575, "nonpadding_tokens": 19542, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3223257879999437, "nonpadding_tokens": 6677, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32131146799997623, "nonpadding_tokens": 15109, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31991069200000766, "nonpadding_tokens": 6183, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32244976100002987, "nonpadding_tokens": 10518, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3192670050000288, "nonpadding_tokens": 17041, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3176906600000393, "nonpadding_tokens": 17765, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31883521999998266, "nonpadding_tokens": 8891, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003757009999389993, "prefetch": {"data_wait_seconds": 0.014052809000943967, "consumed_batches": 1500, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2050, "tokens": 154185998, "loss_tokens": 79009, "loss": 2.319660972167728, "lr": 0.0002995166075255911, "optimizer_lrs": [0.019967773835039407, 0.0002995166075255911], "gradient_norm": 0.40262851119041443, "seconds": 0.303610735999996, "nonpadding_tokens_per_second": 260231.2455775643, "ranks": [{"compute_seconds": 0.3008943040000531, "nonpadding_tokens": 2189, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2980198330000121, "nonpadding_tokens": 14922, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2983918380000432, "nonpadding_tokens": 16857, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2989834109999947, "nonpadding_tokens": 3503, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3013030429999617, "nonpadding_tokens": 9851, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30028045200003817, "nonpadding_tokens": 13096, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2972748759999604, "nonpadding_tokens": 8359, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29784090499993, "nonpadding_tokens": 10232, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039420100006282155, "prefetch": {"data_wait_seconds": 0.014061877000997924, "consumed_batches": 1501, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2051, "tokens": 154261100, "loss_tokens": 75102, "loss": 2.835050997310325, "lr": 0.0002995160666019441, "optimizer_lrs": [0.019967737773462942, 0.0002995160666019441], "gradient_norm": 0.3770694136619568, "seconds": 0.23148295199996483, "nonpadding_tokens_per_second": 324438.57895855507, "ranks": [{"compute_seconds": 0.2266049859999839, "nonpadding_tokens": 3791, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22516051299999162, "nonpadding_tokens": 1984, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22611643499999445, "nonpadding_tokens": 12038, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22877391299994088, "nonpadding_tokens": 13859, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2292384590000438, "nonpadding_tokens": 16172, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22757846900003642, "nonpadding_tokens": 16228, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22494472200003202, "nonpadding_tokens": 5081, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22586082199995872, "nonpadding_tokens": 5949, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034436999999343243, "prefetch": {"data_wait_seconds": 0.01407086500098842, "consumed_batches": 1502, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2052, "tokens": 154322258, "loss_tokens": 61158, "loss": 2.1232012522891526, "lr": 0.0002995156258871817, "optimizer_lrs": [0.01996770839247878, 0.0002995156258871817], "gradient_norm": 0.4207920432090759, "seconds": 0.2496209750000844, "nonpadding_tokens_per_second": 245003.44973005302, "ranks": [{"compute_seconds": 0.24386613100000432, "nonpadding_tokens": 5338, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24664669300000241, "nonpadding_tokens": 15893, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24328899299996465, "nonpadding_tokens": 5796, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24629953800001658, "nonpadding_tokens": 7272, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24279540200006977, "nonpadding_tokens": 16649, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24513810599989938, "nonpadding_tokens": 4927, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24220960299999206, "nonpadding_tokens": 1632, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2472345400000222, "nonpadding_tokens": 3651, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037497499999972206, "prefetch": {"data_wait_seconds": 0.01407921800098677, "consumed_batches": 1503, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2053, "tokens": 154370440, "loss_tokens": 48182, "loss": 2.7788226801502636, "lr": 0.0002995152785386332, "optimizer_lrs": [0.01996768523590888, 0.0002995152785386332], "gradient_norm": 0.4678317904472351, "seconds": 0.26375618600002326, "nonpadding_tokens_per_second": 182676.2842255982, "ranks": [{"compute_seconds": 0.25828633799994805, "nonpadding_tokens": 10826, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2607780470000307, "nonpadding_tokens": 7483, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25782425399995645, "nonpadding_tokens": 1875, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26149264700006825, "nonpadding_tokens": 6574, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25638828600006036, "nonpadding_tokens": 5244, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2595472929999687, "nonpadding_tokens": 2392, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2564890890000697, "nonpadding_tokens": 4506, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2573736150000059, "nonpadding_tokens": 9282, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034550600003058207, "prefetch": {"data_wait_seconds": 0.014088175000964043, "consumed_batches": 1504, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2054, "tokens": 154425693, "loss_tokens": 55253, "loss": 2.41050457667457, "lr": 0.0002995148800616138, "optimizer_lrs": [0.019967658670774255, 0.0002995148800616138], "gradient_norm": 0.5336169600486755, "seconds": 0.2528978919999645, "nonpadding_tokens_per_second": 218479.4802481302, "ranks": [{"compute_seconds": 0.24805861100003312, "nonpadding_tokens": 3022, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24837274900005468, "nonpadding_tokens": 5824, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24469234999992295, "nonpadding_tokens": 9273, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25012660699997014, "nonpadding_tokens": 11615, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24360112599993045, "nonpadding_tokens": 10388, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24673806300006618, "nonpadding_tokens": 3963, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24388064900006157, "nonpadding_tokens": 4578, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2447057650000488, "nonpadding_tokens": 6590, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033376399994722306, "prefetch": {"data_wait_seconds": 0.01409569500090413, "consumed_batches": 1505, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2055, "tokens": 154468104, "loss_tokens": 42411, "loss": 2.2705275459196907, "lr": 0.0002995145740884632, "optimizer_lrs": [0.019967638272564214, 0.0002995145740884632], "gradient_norm": 0.395439475774765, "seconds": 0.29396142199993847, "nonpadding_tokens_per_second": 144274.0333458071, "ranks": [{"compute_seconds": 0.2903744349999897, "nonpadding_tokens": 3917, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2916522390000864, "nonpadding_tokens": 2767, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28862733100004334, "nonpadding_tokens": 5513, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29143994000003204, "nonpadding_tokens": 3069, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2878565390000176, "nonpadding_tokens": 12657, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2896720979999827, "nonpadding_tokens": 2458, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28798008299997946, "nonpadding_tokens": 4005, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2881892110000308, "nonpadding_tokens": 8025, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040623000006689836, "prefetch": {"data_wait_seconds": 0.014105567000981978, "consumed_batches": 1506, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2056, "tokens": 154525898, "loss_tokens": 57794, "loss": 2.1807489207357165, "lr": 0.00029951415698001755, "optimizer_lrs": [0.019967610465334504, 0.00029951415698001755], "gradient_norm": 0.5212636590003967, "seconds": 0.2277805050000552, "nonpadding_tokens_per_second": 253726.71818418347, "ranks": [{"compute_seconds": 0.22239390300001105, "nonpadding_tokens": 4799, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2245623380000552, "nonpadding_tokens": 6374, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2213646749999043, "nonpadding_tokens": 3873, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22264680499995393, "nonpadding_tokens": 9693, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2204756470000575, "nonpadding_tokens": 3827, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22173650299998826, "nonpadding_tokens": 11421, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.220067384999993, "nonpadding_tokens": 8885, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2254152629999453, "nonpadding_tokens": 8922, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003871579999668029, "prefetch": {"data_wait_seconds": 0.014113525001064886, "consumed_batches": 1507, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2057, "tokens": 154583885, "loss_tokens": 57987, "loss": 2.455366612344146, "lr": 0.0002995137382989608, "optimizer_lrs": [0.019967582553264057, 0.0002995137382989608], "gradient_norm": 1.3562262058258057, "seconds": 0.2067059319999771, "nonpadding_tokens_per_second": 280528.95937213075, "ranks": [{"compute_seconds": 0.2043891270000131, "nonpadding_tokens": 5783, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20378618400002324, "nonpadding_tokens": 4061, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2036871699999665, "nonpadding_tokens": 2405, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2044553810000025, "nonpadding_tokens": 5204, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20313763200010726, "nonpadding_tokens": 2157, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20408664199999293, "nonpadding_tokens": 5364, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20302549799998815, "nonpadding_tokens": 15128, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2041957400000456, "nonpadding_tokens": 17885, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039102099992760486, "prefetch": {"data_wait_seconds": 0.014121564001086995, "consumed_batches": 1508, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2058, "tokens": 154647127, "loss_tokens": 63242, "loss": 2.735032197748332, "lr": 0.0002995132814702464, "optimizer_lrs": [0.019967552098016428, 0.0002995132814702464], "gradient_norm": 0.49853286147117615, "seconds": 0.24283784500005368, "nonpadding_tokens_per_second": 260428.92943637355, "ranks": [{"compute_seconds": 0.23892816200009293, "nonpadding_tokens": 2947, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24033492800003842, "nonpadding_tokens": 10456, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23771031499995843, "nonpadding_tokens": 20908, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2390859319999663, "nonpadding_tokens": 9574, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23732156099993063, "nonpadding_tokens": 3179, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2368719419999934, "nonpadding_tokens": 4049, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23653400900002453, "nonpadding_tokens": 2391, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23982245000001967, "nonpadding_tokens": 9738, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005772279999973762, "prefetch": {"data_wait_seconds": 0.01413016700116998, "consumed_batches": 1509, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2059, "tokens": 154701549, "loss_tokens": 54422, "loss": 2.2806028582191025, "lr": 0.00029951288818142623, "optimizer_lrs": [0.01996752587876175, 0.00029951288818142623], "gradient_norm": 0.40343254804611206, "seconds": 0.2889386019999165, "nonpadding_tokens_per_second": 188351.43391472395, "ranks": [{"compute_seconds": 0.2848869219999415, "nonpadding_tokens": 4730, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28610121199994865, "nonpadding_tokens": 3386, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2860223559999895, "nonpadding_tokens": 4958, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2860916440000665, "nonpadding_tokens": 15820, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28195895800001836, "nonpadding_tokens": 2371, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28518830799998796, "nonpadding_tokens": 4011, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2865186569999878, "nonpadding_tokens": 5341, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28299885199999153, "nonpadding_tokens": 13805, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038834099996165605, "prefetch": {"data_wait_seconds": 0.014138918001165734, "consumed_batches": 1510, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2060, "tokens": 154769916, "loss_tokens": 68367, "loss": 2.574244518554273, "lr": 0.00029951239389226444, "optimizer_lrs": [0.019967492926150965, 0.00029951239389226444], "gradient_norm": 0.38661888241767883, "seconds": 0.3242435189999924, "nonpadding_tokens_per_second": 210850.78341998134, "ranks": [{"compute_seconds": 0.32152382000003854, "nonpadding_tokens": 16986, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31781458900002235, "nonpadding_tokens": 11183, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3202419079999572, "nonpadding_tokens": 6171, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3186733009999898, "nonpadding_tokens": 5554, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3172045539999999, "nonpadding_tokens": 12055, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3198958660000244, "nonpadding_tokens": 9995, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32101179400001456, "nonpadding_tokens": 2293, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3218632239999124, "nonpadding_tokens": 4130, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004426619999549075, "prefetch": {"data_wait_seconds": 0.014149666001188166, "consumed_batches": 1511, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2061, "tokens": 154825623, "loss_tokens": 55707, "loss": 2.463980626312672, "lr": 0.00029951199094911923, "optimizer_lrs": [0.019967466063274618, 0.00029951199094911923], "gradient_norm": 0.48665982484817505, "seconds": 0.24749958300003527, "nonpadding_tokens_per_second": 225079.16710304946, "ranks": [{"compute_seconds": 0.24253321100002267, "nonpadding_tokens": 2468, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24456121299999722, "nonpadding_tokens": 3677, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24176532899991798, "nonpadding_tokens": 5881, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24436867100007476, "nonpadding_tokens": 10480, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24093858699995963, "nonpadding_tokens": 9652, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24327572300001066, "nonpadding_tokens": 7359, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2447628860001032, "nonpadding_tokens": 8228, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24519113600001674, "nonpadding_tokens": 7962, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037981900004524505, "prefetch": {"data_wait_seconds": 0.014159003001239512, "consumed_batches": 1512, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2062, "tokens": 154868603, "loss_tokens": 42980, "loss": 2.3241551302931596, "lr": 0.0002995116799501313, "optimizer_lrs": [0.019967445330008753, 0.0002995116799501313], "gradient_norm": 0.4133482873439789, "seconds": 0.36621174000003975, "nonpadding_tokens_per_second": 117363.79614699227, "ranks": [{"compute_seconds": 0.36135166500002924, "nonpadding_tokens": 6031, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35990555700004734, "nonpadding_tokens": 2929, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3625172580000253, "nonpadding_tokens": 9635, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3609208459999991, "nonpadding_tokens": 3742, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3638857180000059, "nonpadding_tokens": 2793, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36240664100000686, "nonpadding_tokens": 4215, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35944551800002955, "nonpadding_tokens": 7939, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36074700500000745, "nonpadding_tokens": 5696, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003764629999523095, "prefetch": {"data_wait_seconds": 0.014166242001195428, "consumed_batches": 1513, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2063, "tokens": 154938452, "loss_tokens": 69849, "loss": 2.0879698975647467, "lr": 0.000299511174318806, "optimizer_lrs": [0.019967411621253733, 0.000299511174318806], "gradient_norm": 0.6443824172019958, "seconds": 0.2881651289999354, "nonpadding_tokens_per_second": 242392.2708566714, "ranks": [{"compute_seconds": 0.28577519700002085, "nonpadding_tokens": 3717, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2821633109999766, "nonpadding_tokens": 14798, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.285045205000074, "nonpadding_tokens": 1484, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2855791560000398, "nonpadding_tokens": 5219, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28186560200003896, "nonpadding_tokens": 3342, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.284797033000018, "nonpadding_tokens": 4033, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28586959200004003, "nonpadding_tokens": 12799, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2825925089999828, "nonpadding_tokens": 24457, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003376790000402252, "prefetch": {"data_wait_seconds": 0.014175861001220369, "consumed_batches": 1514, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2064, "tokens": 155010273, "loss_tokens": 71821, "loss": 2.161788508931928, "lr": 0.0002995106541399984, "optimizer_lrs": [0.01996737694266656, 0.0002995106541399984], "gradient_norm": 0.42451927065849304, "seconds": 0.5730923450000773, "nonpadding_tokens_per_second": 125321.86239547539, "ranks": [{"compute_seconds": 0.5683260729999802, "nonpadding_tokens": 1663, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5707834190000085, "nonpadding_tokens": 9291, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.5696536480000987, "nonpadding_tokens": 17626, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.5699474340000279, "nonpadding_tokens": 3803, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.5668030809999891, "nonpadding_tokens": 7988, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5692334330000222, "nonpadding_tokens": 7185, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.5704857959999572, "nonpadding_tokens": 5993, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.5668540609999582, "nonpadding_tokens": 18272, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003974319999997533, "prefetch": {"data_wait_seconds": 0.014186451001208411, "consumed_batches": 1515, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2065, "tokens": 155096941, "loss_tokens": 86668, "loss": 2.658167338290949, "lr": 0.00029951002606100563, "optimizer_lrs": [0.01996733507073371, 0.00029951002606100563], "gradient_norm": 0.43073421716690063, "seconds": 0.27064967899991643, "nonpadding_tokens_per_second": 320222.06832185737, "ranks": [{"compute_seconds": 0.2672909980000213, "nonpadding_tokens": 10011, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2671776709998994, "nonpadding_tokens": 12572, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26396492100002433, "nonpadding_tokens": 8743, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26748823599996285, "nonpadding_tokens": 10939, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26825408199999856, "nonpadding_tokens": 17196, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2658720029999131, "nonpadding_tokens": 2821, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2679981979999866, "nonpadding_tokens": 12652, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2635841480000636, "nonpadding_tokens": 11734, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003514339999810545, "prefetch": {"data_wait_seconds": 0.014195106001196223, "consumed_batches": 1516, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2066, "tokens": 155166546, "loss_tokens": 69605, "loss": 2.1961892105452194, "lr": 0.00029950952134565355, "optimizer_lrs": [0.019967301423043572, 0.00029950952134565355], "gradient_norm": 0.4510100781917572, "seconds": 0.37372916799995437, "nonpadding_tokens_per_second": 186244.49456941636, "ranks": [{"compute_seconds": 0.36871958200003974, "nonpadding_tokens": 9540, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37124195000001237, "nonpadding_tokens": 5058, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37050711599999886, "nonpadding_tokens": 4516, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3707970739999382, "nonpadding_tokens": 6018, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.366780163000044, "nonpadding_tokens": 5607, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37073959500003184, "nonpadding_tokens": 16938, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3713694969999324, "nonpadding_tokens": 1786, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3677272569999559, "nonpadding_tokens": 20142, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003854180000644192, "prefetch": {"data_wait_seconds": 0.014204798001244967, "consumed_batches": 1517, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2067, "tokens": 155251041, "loss_tokens": 84495, "loss": 2.1464618838392804, "lr": 0.00029950890831238395, "optimizer_lrs": [0.01996726055415893, 0.00029950890831238395], "gradient_norm": 0.3974914252758026, "seconds": 0.2656609330000492, "nonpadding_tokens_per_second": 318055.79783906107, "ranks": [{"compute_seconds": 0.26063454899997396, "nonpadding_tokens": 10620, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2621527310000147, "nonpadding_tokens": 22576, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26197823999996217, "nonpadding_tokens": 14816, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25965765299997656, "nonpadding_tokens": 5244, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25843004299997574, "nonpadding_tokens": 4906, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2605350080000335, "nonpadding_tokens": 3260, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2633307049999303, "nonpadding_tokens": 11406, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25839097200002925, "nonpadding_tokens": 11667, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033802199993715476, "prefetch": {"data_wait_seconds": 0.014213521001195772, "consumed_batches": 1518, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2068, "tokens": 155368607, "loss_tokens": 117566, "loss": 2.3196005328921627, "lr": 0.00029950805470457304, "optimizer_lrs": [0.019967203646971538, 0.00029950805470457304], "gradient_norm": 0.3822830617427826, "seconds": 0.2522133260000601, "nonpadding_tokens_per_second": 466137.14613942325, "ranks": [{"compute_seconds": 0.24861255500002244, "nonpadding_tokens": 11389, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2486219180000262, "nonpadding_tokens": 17513, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2453928109999879, "nonpadding_tokens": 20142, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24885979399994085, "nonpadding_tokens": 19114, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24382730799993624, "nonpadding_tokens": 8669, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2467295249999779, "nonpadding_tokens": 3962, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2435694790000298, "nonpadding_tokens": 18354, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2499014239999724, "nonpadding_tokens": 18423, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003650600000355553, "prefetch": {"data_wait_seconds": 0.014222311001120147, "consumed_batches": 1519, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2069, "tokens": 155442020, "loss_tokens": 73413, "loss": 2.475375827510114, "lr": 0.0002995075213018323, "optimizer_lrs": [0.01996716808678882, 0.0002995075213018323], "gradient_norm": 0.3565439283847809, "seconds": 0.3301679899999499, "nonpadding_tokens_per_second": 222350.4465106116, "ranks": [{"compute_seconds": 0.32723375800003396, "nonpadding_tokens": 10362, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32725393700002314, "nonpadding_tokens": 12051, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3241295809999656, "nonpadding_tokens": 7024, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32699252099996556, "nonpadding_tokens": 13588, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32784693500002504, "nonpadding_tokens": 6105, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3261941770000476, "nonpadding_tokens": 2825, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3269593600000462, "nonpadding_tokens": 17276, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32774603600000773, "nonpadding_tokens": 4182, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004051739999795245, "prefetch": {"data_wait_seconds": 0.014231195001187302, "consumed_batches": 1520, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2070, "tokens": 155506603, "loss_tokens": 64583, "loss": 2.446402545948624, "lr": 0.00029950705181740325, "optimizer_lrs": [0.019967136787826884, 0.00029950705181740325], "gradient_norm": 1.0681228637695312, "seconds": 0.26982575099998485, "nonpadding_tokens_per_second": 239350.76530187673, "ranks": [{"compute_seconds": 0.2662239950000185, "nonpadding_tokens": 10553, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2670307099999718, "nonpadding_tokens": 4725, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26590884799998094, "nonpadding_tokens": 14112, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2644828799999459, "nonpadding_tokens": 5538, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26701158900004884, "nonpadding_tokens": 6533, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2655836349999845, "nonpadding_tokens": 9652, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2665144220000002, "nonpadding_tokens": 2559, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2674615849999782, "nonpadding_tokens": 10911, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004149260000758659, "prefetch": {"data_wait_seconds": 0.014241747001122462, "consumed_batches": 1521, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2071, "tokens": 155639194, "loss_tokens": 132591, "loss": 2.2136690555920087, "lr": 0.0002995060872508854, "optimizer_lrs": [0.01996707248339236, 0.0002995060872508854], "gradient_norm": 4.029052257537842, "seconds": 0.2765778269999828, "nonpadding_tokens_per_second": 479398.5166425082, "ranks": [{"compute_seconds": 0.2720448630000192, "nonpadding_tokens": 11651, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27053737999995064, "nonpadding_tokens": 18018, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27318974900003923, "nonpadding_tokens": 13682, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2715577529999109, "nonpadding_tokens": 22434, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27421950199993717, "nonpadding_tokens": 20086, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27282786300008866, "nonpadding_tokens": 15347, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27022450300000855, "nonpadding_tokens": 12503, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2708583770000814, "nonpadding_tokens": 18870, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000427983999998105, "prefetch": {"data_wait_seconds": 0.01424969900104145, "consumed_batches": 1522, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2072, "tokens": 155740529, "loss_tokens": 101335, "loss": 2.3592447081462478, "lr": 0.0002995053494295458, "optimizer_lrs": [0.019967023295303053, 0.0002995053494295458], "gradient_norm": 1.570271372795105, "seconds": 0.41363046300000406, "nonpadding_tokens_per_second": 244989.20912408477, "ranks": [{"compute_seconds": 0.4102707619999819, "nonpadding_tokens": 19724, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4113040249999358, "nonpadding_tokens": 10267, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4079776580000498, "nonpadding_tokens": 10316, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4105198519999931, "nonpadding_tokens": 6321, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.40702484799999183, "nonpadding_tokens": 17088, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4098764509999455, "nonpadding_tokens": 4229, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.41062183099995764, "nonpadding_tokens": 14129, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4071600590000344, "nonpadding_tokens": 19261, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037057899999126676, "prefetch": {"data_wait_seconds": 0.014257910001063578, "consumed_batches": 1523, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2073, "tokens": 155823396, "loss_tokens": 82867, "loss": 2.5515681377991237, "lr": 0.00029950474566540114, "optimizer_lrs": [0.01996698304436008, 0.00029950474566540114], "gradient_norm": 0.6270527839660645, "seconds": 0.25143787000001794, "nonpadding_tokens_per_second": 329572.4705271886, "ranks": [{"compute_seconds": 0.24752495199993518, "nonpadding_tokens": 4771, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24472613600005388, "nonpadding_tokens": 4458, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24528204700004608, "nonpadding_tokens": 9228, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2462789869999824, "nonpadding_tokens": 2343, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24454138500004774, "nonpadding_tokens": 27050, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2458222709999518, "nonpadding_tokens": 19077, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2481791319999047, "nonpadding_tokens": 13367, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2491375380000136, "nonpadding_tokens": 2573, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003893770000331642, "prefetch": {"data_wait_seconds": 0.014267258001112282, "consumed_batches": 1524, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2074, "tokens": 155884453, "loss_tokens": 61057, "loss": 2.5043284349050885, "lr": 0.00029950430057245925, "optimizer_lrs": [0.019966953371497283, 0.00029950430057245925], "gradient_norm": 0.43860214948654175, "seconds": 0.2808451600000126, "nonpadding_tokens_per_second": 217404.4943484063, "ranks": [{"compute_seconds": 0.27762855200001013, "nonpadding_tokens": 18767, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27809873199998947, "nonpadding_tokens": 3290, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27469039100003556, "nonpadding_tokens": 11539, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2772670650000464, "nonpadding_tokens": 8081, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27374329399992803, "nonpadding_tokens": 2623, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27499184999999216, "nonpadding_tokens": 8220, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27777190700010124, "nonpadding_tokens": 5382, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2784877469999856, "nonpadding_tokens": 3155, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003499219999412162, "prefetch": {"data_wait_seconds": 0.014275867001131815, "consumed_batches": 1525, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2075, "tokens": 155933531, "loss_tokens": 49078, "loss": 2.3358975444700274, "lr": 0.00029950394265929395, "optimizer_lrs": [0.019966929510619597, 0.00029950394265929395], "gradient_norm": 0.5572695732116699, "seconds": 0.30878695799992784, "nonpadding_tokens_per_second": 158938.0598127835, "ranks": [{"compute_seconds": 0.3051636970000118, "nonpadding_tokens": 6548, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3062363039999809, "nonpadding_tokens": 4860, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3032497899999953, "nonpadding_tokens": 10768, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3036413919999177, "nonpadding_tokens": 2447, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3023854079999637, "nonpadding_tokens": 6117, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30493777399999544, "nonpadding_tokens": 8257, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30252261600003294, "nonpadding_tokens": 4134, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.306413578000047, "nonpadding_tokens": 5947, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040594700010387896, "prefetch": {"data_wait_seconds": 0.014284037001132788, "consumed_batches": 1526, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2076, "tokens": 156016239, "loss_tokens": 82708, "loss": 2.4966912964888524, "lr": 0.0002995033391995159, "optimizer_lrs": [0.01996688927996773, 0.0002995033391995159], "gradient_norm": 0.40996310114860535, "seconds": 0.3007066530000202, "nonpadding_tokens_per_second": 275045.46099947597, "ranks": [{"compute_seconds": 0.29790548899995883, "nonpadding_tokens": 10929, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29551884499994685, "nonpadding_tokens": 2265, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2979361699999572, "nonpadding_tokens": 3772, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2983969760000491, "nonpadding_tokens": 5983, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29493466799999624, "nonpadding_tokens": 15261, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2982299880000028, "nonpadding_tokens": 10562, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29509444300003906, "nonpadding_tokens": 20430, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29571132000000944, "nonpadding_tokens": 13506, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003861159999587471, "prefetch": {"data_wait_seconds": 0.014293478001107474, "consumed_batches": 1527, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2077, "tokens": 156060711, "loss_tokens": 44472, "loss": 2.2844551417183845, "lr": 0.000299503014568488, "optimizer_lrs": [0.0199668676378992, 0.000299503014568488], "gradient_norm": 0.7338553071022034, "seconds": 0.2679877340000303, "nonpadding_tokens_per_second": 165947.8937196244, "ranks": [{"compute_seconds": 0.26531101999989914, "nonpadding_tokens": 4835, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26568203799990897, "nonpadding_tokens": 3388, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.262555852000105, "nonpadding_tokens": 3463, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2648567330001015, "nonpadding_tokens": 6311, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2617430120000108, "nonpadding_tokens": 10169, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26284952200001044, "nonpadding_tokens": 5950, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2613508509999747, "nonpadding_tokens": 7297, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2624812720000591, "nonpadding_tokens": 3059, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003540249999787193, "prefetch": {"data_wait_seconds": 0.014302850001129741, "consumed_batches": 1528, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2078, "tokens": 156110670, "loss_tokens": 49959, "loss": 2.883304622290278, "lr": 0.0002995026497578965, "optimizer_lrs": [0.019966843317193103, 0.0002995026497578965], "gradient_norm": 0.46977314352989197, "seconds": 0.24266933699993842, "nonpadding_tokens_per_second": 205872.73455159552, "ranks": [{"compute_seconds": 0.24033650400008355, "nonpadding_tokens": 6304, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24000316399997246, "nonpadding_tokens": 4372, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23722871799998302, "nonpadding_tokens": 7645, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23971064600004865, "nonpadding_tokens": 2841, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23595602000000326, "nonpadding_tokens": 9669, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23880238200001713, "nonpadding_tokens": 10943, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23578854200002297, "nonpadding_tokens": 1392, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24030169599996043, "nonpadding_tokens": 6793, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035671800003456156, "prefetch": {"data_wait_seconds": 0.014311606001186306, "consumed_batches": 1529, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2079, "tokens": 156191709, "loss_tokens": 81039, "loss": 2.784731926911734, "lr": 0.0002995020577108212, "optimizer_lrs": [0.01996680384738808, 0.0002995020577108212], "gradient_norm": 0.3819361627101898, "seconds": 0.2445660460000454, "nonpadding_tokens_per_second": 331358.3439950816, "ranks": [{"compute_seconds": 0.24215229799995086, "nonpadding_tokens": 3161, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2385583880000013, "nonpadding_tokens": 10656, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23855997299995124, "nonpadding_tokens": 13156, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24117247000003772, "nonpadding_tokens": 19583, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24229316700007075, "nonpadding_tokens": 11726, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2404017420000173, "nonpadding_tokens": 3548, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23767199599990363, "nonpadding_tokens": 8552, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23873660300000665, "nonpadding_tokens": 10657, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035447300001578697, "prefetch": {"data_wait_seconds": 0.014320641001248191, "consumed_batches": 1530, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2080, "tokens": 156244634, "loss_tokens": 52925, "loss": 2.469404522909778, "lr": 0.00029950167086659966, "optimizer_lrs": [0.01996677805777331, 0.00029950167086659966], "gradient_norm": 0.38963133096694946, "seconds": 0.2664370779999672, "nonpadding_tokens_per_second": 198639.77040014873, "ranks": [{"compute_seconds": 0.2618403789999775, "nonpadding_tokens": 3532, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2638601060000383, "nonpadding_tokens": 2602, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2610818670000299, "nonpadding_tokens": 10963, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2634146589999773, "nonpadding_tokens": 2516, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2602728439999282, "nonpadding_tokens": 6112, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2609094829999776, "nonpadding_tokens": 3935, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2638243320000129, "nonpadding_tokens": 9549, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.264126802000078, "nonpadding_tokens": 13716, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040077600010590686, "prefetch": {"data_wait_seconds": 0.014329151001291507, "consumed_batches": 1531, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2081, "tokens": 156312349, "loss_tokens": 67715, "loss": 2.4247311803145535, "lr": 0.00029950117569933294, "optimizer_lrs": [0.019966745046622197, 0.00029950117569933294], "gradient_norm": 0.3806145489215851, "seconds": 0.31365246800010027, "nonpadding_tokens_per_second": 215891.8131005376, "ranks": [{"compute_seconds": 0.31146458499995333, "nonpadding_tokens": 14621, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3114042980000704, "nonpadding_tokens": 2908, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31025672500004475, "nonpadding_tokens": 11485, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3091581449999694, "nonpadding_tokens": 4957, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3075810880000063, "nonpadding_tokens": 15176, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31023238699992817, "nonpadding_tokens": 3755, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3074633769999764, "nonpadding_tokens": 5546, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30842655200001445, "nonpadding_tokens": 9267, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035214399997585133, "prefetch": {"data_wait_seconds": 0.014337552001350673, "consumed_batches": 1532, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2082, "tokens": 156379206, "loss_tokens": 66857, "loss": 2.0880592159384954, "lr": 0.00029950068656541787, "optimizer_lrs": [0.019966712437694527, 0.00029950068656541787], "gradient_norm": 0.43424201011657715, "seconds": 0.23453384000004007, "nonpadding_tokens_per_second": 285063.340966014, "ranks": [{"compute_seconds": 0.23219695700004195, "nonpadding_tokens": 5754, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2320816010000044, "nonpadding_tokens": 13887, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2307914160001019, "nonpadding_tokens": 7276, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2316378689999965, "nonpadding_tokens": 12497, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22811831599995003, "nonpadding_tokens": 3015, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22963660000004893, "nonpadding_tokens": 1325, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2280549739999742, "nonpadding_tokens": 11355, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22859579800001484, "nonpadding_tokens": 11748, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003945400000020527, "prefetch": {"data_wait_seconds": 0.014347199001306876, "consumed_batches": 1533, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2083, "tokens": 156416296, "loss_tokens": 37090, "loss": 2.583421660150984, "lr": 0.0002995004151073004, "optimizer_lrs": [0.019966694340486695, 0.0002995004151073004], "gradient_norm": 0.44796913862228394, "seconds": 0.38489741399996547, "nonpadding_tokens_per_second": 96363.33903766713, "ranks": [{"compute_seconds": 0.38256430000001274, "nonpadding_tokens": 2519, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37840857799994865, "nonpadding_tokens": 4414, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3809025319999364, "nonpadding_tokens": 10066, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3792249030000221, "nonpadding_tokens": 6616, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3819625520000045, "nonpadding_tokens": 2325, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3803792430001067, "nonpadding_tokens": 5830, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.381435035000095, "nonpadding_tokens": 2157, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3789573550000114, "nonpadding_tokens": 3163, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004122640000332467, "prefetch": {"data_wait_seconds": 0.014356127001363461, "consumed_batches": 1534, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2084, "tokens": 156491163, "loss_tokens": 74867, "loss": 2.3157526430202893, "lr": 0.00029949986693865487, "optimizer_lrs": [0.019966657795910326, 0.00029949986693865487], "gradient_norm": 0.3708168864250183, "seconds": 0.28963179899994884, "nonpadding_tokens_per_second": 258490.26335679812, "ranks": [{"compute_seconds": 0.2854153260000203, "nonpadding_tokens": 8119, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28696096300006957, "nonpadding_tokens": 3660, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28447865999999067, "nonpadding_tokens": 7178, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.284685890999981, "nonpadding_tokens": 6751, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28401764900002036, "nonpadding_tokens": 17635, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28602060000002894, "nonpadding_tokens": 15283, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2838971549999769, "nonpadding_tokens": 13621, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28487716300003285, "nonpadding_tokens": 2620, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003681649999407455, "prefetch": {"data_wait_seconds": 0.01436512300138304, "consumed_batches": 1535, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2085, "tokens": 156550885, "loss_tokens": 59722, "loss": 2.72236178669502, "lr": 0.0002994994294450583, "optimizer_lrs": [0.019966628629670555, 0.0002994994294450583], "gradient_norm": 0.3981485366821289, "seconds": 0.27042165000000296, "nonpadding_tokens_per_second": 220847.7020978141, "ranks": [{"compute_seconds": 0.2681836579999981, "nonpadding_tokens": 6492, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2649818210001058, "nonpadding_tokens": 19767, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2662360980000358, "nonpadding_tokens": 7493, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26304866599991783, "nonpadding_tokens": 1480, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26096089899999697, "nonpadding_tokens": 9658, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26377439400005187, "nonpadding_tokens": 2652, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2670188680000365, "nonpadding_tokens": 3977, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26152938000007, "nonpadding_tokens": 8203, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033956699996906536, "prefetch": {"data_wait_seconds": 0.01437435400141851, "consumed_batches": 1536, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2086, "tokens": 156590345, "loss_tokens": 39460, "loss": 2.3118808999619866, "lr": 0.00029949914027602956, "optimizer_lrs": [0.019966609351735304, 0.00029949914027602956], "gradient_norm": 0.44546860456466675, "seconds": 0.21321015599994553, "nonpadding_tokens_per_second": 185075.6115014056, "ranks": [{"compute_seconds": 0.2077728730000672, "nonpadding_tokens": 2820, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20985718700001144, "nonpadding_tokens": 2794, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2069389330000604, "nonpadding_tokens": 4609, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2073646579999604, "nonpadding_tokens": 7196, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20580852799992044, "nonpadding_tokens": 9103, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20852442000000337, "nonpadding_tokens": 3592, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2057055450000007, "nonpadding_tokens": 2296, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21083833499994853, "nonpadding_tokens": 7050, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003830699999980425, "prefetch": {"data_wait_seconds": 0.014382281001374508, "consumed_batches": 1537, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2087, "tokens": 156696529, "loss_tokens": 106184, "loss": 2.4261282302418445, "lr": 0.00029949836172924485, "optimizer_lrs": [0.019966557448616324, 0.00029949836172924485], "gradient_norm": 0.3634741008281708, "seconds": 0.30290477799997007, "nonpadding_tokens_per_second": 350552.4102363631, "ranks": [{"compute_seconds": 0.29891393200000493, "nonpadding_tokens": 15663, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29984343700004956, "nonpadding_tokens": 12965, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29741338199994516, "nonpadding_tokens": 11610, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2975692219999928, "nonpadding_tokens": 12448, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30056036300004507, "nonpadding_tokens": 18770, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29913870699999734, "nonpadding_tokens": 10830, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29616444200007663, "nonpadding_tokens": 18979, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30058785699998225, "nonpadding_tokens": 4919, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034033399992949853, "prefetch": {"data_wait_seconds": 0.014390738001409886, "consumed_batches": 1538, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2088, "tokens": 156750839, "loss_tokens": 54310, "loss": 2.2344194496869823, "lr": 0.0002994979632921368, "optimizer_lrs": [0.019966530886142454, 0.0002994979632921368], "gradient_norm": 0.4171936511993408, "seconds": 0.323690436999982, "nonpadding_tokens_per_second": 167783.76433778615, "ranks": [{"compute_seconds": 0.31896053699995264, "nonpadding_tokens": 6000, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3210884710000528, "nonpadding_tokens": 2017, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32029833800004326, "nonpadding_tokens": 2359, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32049900900005923, "nonpadding_tokens": 5971, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3211013179999327, "nonpadding_tokens": 14781, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3199921110000332, "nonpadding_tokens": 4702, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3170430479999595, "nonpadding_tokens": 11633, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3213846610000246, "nonpadding_tokens": 6847, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033927699996638694, "prefetch": {"data_wait_seconds": 0.014398981001434663, "consumed_batches": 1539, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2089, "tokens": 156803092, "loss_tokens": 52253, "loss": 2.4849337298815377, "lr": 0.0002994975797968784, "optimizer_lrs": [0.019966505319791893, 0.0002994975797968784], "gradient_norm": 0.38734737038612366, "seconds": 0.2445527499999116, "nonpadding_tokens_per_second": 213667.60341079332, "ranks": [{"compute_seconds": 0.24167526099995484, "nonpadding_tokens": 2970, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23802454099995884, "nonpadding_tokens": 14589, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24027811600001314, "nonpadding_tokens": 2654, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2396082390000629, "nonpadding_tokens": 2560, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23805379899999934, "nonpadding_tokens": 3086, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24055457400004343, "nonpadding_tokens": 3001, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23787081000000398, "nonpadding_tokens": 15468, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2421856740000976, "nonpadding_tokens": 7925, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003769659999761643, "prefetch": {"data_wait_seconds": 0.014407338001390713, "consumed_batches": 1540, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2090, "tokens": 156848780, "loss_tokens": 45688, "loss": 2.064713040076169, "lr": 0.00029949724436371846, "optimizer_lrs": [0.019966482957581234, 0.00029949724436371846], "gradient_norm": 0.43273627758026123, "seconds": 0.26572552000004634, "nonpadding_tokens_per_second": 171936.81660682056, "ranks": [{"compute_seconds": 0.26253251999992244, "nonpadding_tokens": 6288, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26336847899995064, "nonpadding_tokens": 1834, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.260326601000088, "nonpadding_tokens": 2093, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26051427100003366, "nonpadding_tokens": 2256, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26319685199996457, "nonpadding_tokens": 12529, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.260560719999944, "nonpadding_tokens": 5649, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2592751519999865, "nonpadding_tokens": 10243, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2634459020000577, "nonpadding_tokens": 4796, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035911199995553034, "prefetch": {"data_wait_seconds": 0.014416800001484376, "consumed_batches": 1541, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2091, "tokens": 156911526, "loss_tokens": 62746, "loss": 2.15891376541931, "lr": 0.0002994967835116746, "optimizer_lrs": [0.019966452234111642, 0.0002994967835116746], "gradient_norm": 0.4318775236606598, "seconds": 0.4187705059999871, "nonpadding_tokens_per_second": 149833.85673298093, "ranks": [{"compute_seconds": 0.41414633499994125, "nonpadding_tokens": 9845, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4158692330000804, "nonpadding_tokens": 15105, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4131477299999915, "nonpadding_tokens": 6705, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4134241199999451, "nonpadding_tokens": 1409, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4163638390000415, "nonpadding_tokens": 3238, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41342756599999575, "nonpadding_tokens": 8046, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4161176100000148, "nonpadding_tokens": 8321, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.41247279999993225, "nonpadding_tokens": 10077, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038088600001628947, "prefetch": {"data_wait_seconds": 0.014426652001546358, "consumed_batches": 1542, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2092, "tokens": 157011256, "loss_tokens": 99730, "loss": 2.482715049383335, "lr": 0.0002994960505887147, "optimizer_lrs": [0.01996640337258098, 0.0002994960505887147], "gradient_norm": 0.4753550589084625, "seconds": 0.3298476899999514, "nonpadding_tokens_per_second": 302351.67025124445, "ranks": [{"compute_seconds": 0.32722977499997796, "nonpadding_tokens": 21334, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32404551100000845, "nonpadding_tokens": 11185, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3236139790000152, "nonpadding_tokens": 18641, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32699925800000074, "nonpadding_tokens": 14958, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32279951600003187, "nonpadding_tokens": 1999, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32588154999996277, "nonpadding_tokens": 18573, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32740494500001205, "nonpadding_tokens": 11271, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32327043700001923, "nonpadding_tokens": 1769, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003544820000342952, "prefetch": {"data_wait_seconds": 0.014435863001565963, "consumed_batches": 1543, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2093, "tokens": 157055335, "loss_tokens": 44079, "loss": 2.418827914086073, "lr": 0.00029949572647932184, "optimizer_lrs": [0.019966381765288123, 0.00029949572647932184], "gradient_norm": 0.4598678946495056, "seconds": 0.2364360519999309, "nonpadding_tokens_per_second": 186430.95935307228, "ranks": [{"compute_seconds": 0.23082961700004034, "nonpadding_tokens": 4172, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23409226000001127, "nonpadding_tokens": 1532, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23047173400004795, "nonpadding_tokens": 4011, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23093684000002668, "nonpadding_tokens": 3562, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2335263129999703, "nonpadding_tokens": 4677, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2324462130000029, "nonpadding_tokens": 9729, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23312400099996466, "nonpadding_tokens": 9156, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23027732099990317, "nonpadding_tokens": 7240, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000413862999948833, "prefetch": {"data_wait_seconds": 0.014445884001546005, "consumed_batches": 1544, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2094, "tokens": 157107812, "loss_tokens": 52477, "loss": 2.449770554242811, "lr": 0.00029949534048449806, "optimizer_lrs": [0.019966356032299872, 0.00029949534048449806], "gradient_norm": 0.40889233350753784, "seconds": 0.2256155850000141, "nonpadding_tokens_per_second": 232594.74738855794, "ranks": [{"compute_seconds": 0.22112617800007683, "nonpadding_tokens": 5051, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21895207599993682, "nonpadding_tokens": 2763, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21975193299999773, "nonpadding_tokens": 2885, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22216877199991814, "nonpadding_tokens": 9813, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21923187599998073, "nonpadding_tokens": 12457, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22158434899995427, "nonpadding_tokens": 5010, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2222880309999482, "nonpadding_tokens": 4051, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2233166839999967, "nonpadding_tokens": 10447, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033561700001882855, "prefetch": {"data_wait_seconds": 0.014454850001541786, "consumed_batches": 1545, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2095, "tokens": 157217130, "loss_tokens": 109318, "loss": 2.360099806299054, "lr": 0.0002994945359220862, "optimizer_lrs": [0.01996630239480575, 0.0002994945359220862], "gradient_norm": 0.37737956643104553, "seconds": 0.2916599520000318, "nonpadding_tokens_per_second": 374813.2002709377, "ranks": [{"compute_seconds": 0.2872811790000469, "nonpadding_tokens": 12938, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2890590129999282, "nonpadding_tokens": 19523, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28634074700005385, "nonpadding_tokens": 2732, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28673036099996807, "nonpadding_tokens": 12293, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28933487899996635, "nonpadding_tokens": 11355, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2865334979999261, "nonpadding_tokens": 22404, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2851885119999906, "nonpadding_tokens": 16564, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28600834399992436, "nonpadding_tokens": 11509, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033880600005886663, "prefetch": {"data_wait_seconds": 0.014462630001574439, "consumed_batches": 1546, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2096, "tokens": 157287541, "loss_tokens": 70411, "loss": 2.6431696574398886, "lr": 0.0002994940173700954, "optimizer_lrs": [0.019966267824673027, 0.0002994940173700954], "gradient_norm": 0.35467034578323364, "seconds": 0.26881611099997826, "nonpadding_tokens_per_second": 261929.98529022574, "ranks": [{"compute_seconds": 0.26620453599991833, "nonpadding_tokens": 3155, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2635480550000011, "nonpadding_tokens": 1744, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2634468009999864, "nonpadding_tokens": 3042, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2664134329999115, "nonpadding_tokens": 8589, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2626992269999846, "nonpadding_tokens": 3742, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2653910149999774, "nonpadding_tokens": 8219, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26647463800009064, "nonpadding_tokens": 25557, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2630060390000608, "nonpadding_tokens": 16363, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003422390000196174, "prefetch": {"data_wait_seconds": 0.01447154500154113, "consumed_batches": 1547, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2097, "tokens": 157357317, "loss_tokens": 69776, "loss": 2.581600630947604, "lr": 0.0002994935032328791, "optimizer_lrs": [0.019966233548858608, 0.0002994935032328791], "gradient_norm": 0.4198138415813446, "seconds": 0.30423636900002293, "nonpadding_tokens_per_second": 229347.99093659554, "ranks": [{"compute_seconds": 0.26190013799998724, "nonpadding_tokens": 2446, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2666522140000325, "nonpadding_tokens": 3959, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2632333720000588, "nonpadding_tokens": 3402, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26629815499995857, "nonpadding_tokens": 16364, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2666872659999626, "nonpadding_tokens": 11851, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26547181199998704, "nonpadding_tokens": 18962, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26245719999997164, "nonpadding_tokens": 6992, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26732525800002804, "nonpadding_tokens": 5800, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01155676299993047, "prefetch": {"data_wait_seconds": 0.01447925400152883, "consumed_batches": 1548, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2098, "tokens": 157403938, "loss_tokens": 46621, "loss": 2.31059048631518, "lr": 0.00029949315956565465, "optimizer_lrs": [0.01996621063771031, 0.00029949315956565465], "gradient_norm": 0.5064961314201355, "seconds": 0.23597600100004001, "nonpadding_tokens_per_second": 197566.70086121213, "ranks": [{"compute_seconds": 0.20562351800003853, "nonpadding_tokens": 3095, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22051442500003304, "nonpadding_tokens": 4705, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21753032399999483, "nonpadding_tokens": 4740, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22335992799992255, "nonpadding_tokens": 2276, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21657176000007894, "nonpadding_tokens": 19484, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21887703599998076, "nonpadding_tokens": 4636, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21644870599993737, "nonpadding_tokens": 3190, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2172865529998944, "nonpadding_tokens": 4495, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000367485000083434, "prefetch": {"data_wait_seconds": 0.014486876001527094, "consumed_batches": 1549, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2099, "tokens": 157447982, "loss_tokens": 44044, "loss": 2.347505832235946, "lr": 0.000299492834787949, "optimizer_lrs": [0.01996618898586327, 0.000299492834787949], "gradient_norm": 0.5063425302505493, "seconds": 0.2777050519999875, "nonpadding_tokens_per_second": 158599.92349005584, "ranks": [{"compute_seconds": 0.2732631900000797, "nonpadding_tokens": 1589, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27537996099999873, "nonpadding_tokens": 2680, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2725712640000211, "nonpadding_tokens": 5609, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27304776500000116, "nonpadding_tokens": 5096, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2736760469999808, "nonpadding_tokens": 15199, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2743088170000192, "nonpadding_tokens": 2298, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27140394399998513, "nonpadding_tokens": 2502, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2744983500000444, "nonpadding_tokens": 9071, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036867400001483475, "prefetch": {"data_wait_seconds": 0.01449618000147268, "consumed_batches": 1550, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2100, "tokens": 157520341, "loss_tokens": 72359, "loss": 2.4475967916914274, "lr": 0.00029949230099179496, "optimizer_lrs": [0.019966153399453, 0.00029949230099179496], "gradient_norm": 0.45575469732284546, "seconds": 0.2945441409999603, "nonpadding_tokens_per_second": 245664.36716189765, "ranks": [{"compute_seconds": 0.2922111649999124, "nonpadding_tokens": 12907, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2882658580000452, "nonpadding_tokens": 18520, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.291365801999973, "nonpadding_tokens": 10787, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29166770099993755, "nonpadding_tokens": 6790, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2881120629999714, "nonpadding_tokens": 3076, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2904646970000613, "nonpadding_tokens": 8294, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2919889179999018, "nonpadding_tokens": 8488, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29221279299997605, "nonpadding_tokens": 3497, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003877800000964271, "prefetch": {"data_wait_seconds": 0.014504500001365273, "consumed_batches": 1551, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2101, "tokens": 157586164, "loss_tokens": 65823, "loss": 2.5545506509882565, "lr": 0.00029949181516864757, "optimizer_lrs": [0.019966121011243173, 0.00029949181516864757], "gradient_norm": 0.4324566721916199, "seconds": 0.37865234799994596, "nonpadding_tokens_per_second": 173834.91835632137, "ranks": [{"compute_seconds": 0.3759042420000469, "nonpadding_tokens": 14500, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37636121499997444, "nonpadding_tokens": 2467, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3735672479999721, "nonpadding_tokens": 10259, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.37603380600000946, "nonpadding_tokens": 1529, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3763393049999877, "nonpadding_tokens": 8176, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37495349099992836, "nonpadding_tokens": 9040, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3723076999999648, "nonpadding_tokens": 14184, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37290656799996214, "nonpadding_tokens": 5668, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003566950000504221, "prefetch": {"data_wait_seconds": 0.01451312600131871, "consumed_batches": 1552, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2102, "tokens": 157637044, "loss_tokens": 50880, "loss": 2.3390773941136005, "lr": 0.000299491439477152, "optimizer_lrs": [0.01996609596514347, 0.000299491439477152], "gradient_norm": 0.5611565113067627, "seconds": 0.25122333299998445, "nonpadding_tokens_per_second": 202528.95856613427, "ranks": [{"compute_seconds": 0.24689917400007744, "nonpadding_tokens": 2749, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2489994029999707, "nonpadding_tokens": 11216, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24871826399999009, "nonpadding_tokens": 6270, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24768699499998093, "nonpadding_tokens": 5690, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24486109600002237, "nonpadding_tokens": 3647, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24570635899999616, "nonpadding_tokens": 9631, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2484823330000836, "nonpadding_tokens": 10040, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24507117799998923, "nonpadding_tokens": 1637, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035681600002135383, "prefetch": {"data_wait_seconds": 0.014522377001370046, "consumed_batches": 1553, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2103, "tokens": 157686338, "loss_tokens": 49294, "loss": 2.360831503326977, "lr": 0.00029949107536433395, "optimizer_lrs": [0.019966071690955597, 0.00029949107536433395], "gradient_norm": 0.41545265913009644, "seconds": 0.28402324800003953, "nonpadding_tokens_per_second": 173556.21537006414, "ranks": [{"compute_seconds": 0.266709934000005, "nonpadding_tokens": 15129, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27762353100001746, "nonpadding_tokens": 10856, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2787144019999914, "nonpadding_tokens": 3375, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27909674399995765, "nonpadding_tokens": 3940, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.277382436000039, "nonpadding_tokens": 3020, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28060028600009446, "nonpadding_tokens": 6018, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2772140350000427, "nonpadding_tokens": 1413, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2817176900000504, "nonpadding_tokens": 5543, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003738160000921198, "prefetch": {"data_wait_seconds": 0.014530489001302271, "consumed_batches": 1554, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2104, "tokens": 157754402, "loss_tokens": 68064, "loss": 2.6944885971291725, "lr": 0.00029949057239212385, "optimizer_lrs": [0.019966038159474925, 0.00029949057239212385], "gradient_norm": 0.4601658582687378, "seconds": 0.25445938300003945, "nonpadding_tokens_per_second": 267484.73252404865, "ranks": [{"compute_seconds": 0.2519723589999785, "nonpadding_tokens": 10209, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24854709999999614, "nonpadding_tokens": 6195, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2512954809999428, "nonpadding_tokens": 3360, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2521190600000409, "nonpadding_tokens": 15339, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.248652753999977, "nonpadding_tokens": 7973, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.251367756000036, "nonpadding_tokens": 19120, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24804730099992867, "nonpadding_tokens": 2424, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24900002599997606, "nonpadding_tokens": 3444, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003809540000929701, "prefetch": {"data_wait_seconds": 0.014539732001210837, "consumed_batches": 1555, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2105, "tokens": 157845916, "loss_tokens": 91514, "loss": 1.7652324712612277, "lr": 0.0002994898957407547, "optimizer_lrs": [0.019965993049383648, 0.0002994898957407547], "gradient_norm": 0.8476579189300537, "seconds": 0.33895584000003964, "nonpadding_tokens_per_second": 269987.9724744949, "ranks": [{"compute_seconds": 0.3355495509999855, "nonpadding_tokens": 7798, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3366484179999816, "nonpadding_tokens": 17691, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3354503450000266, "nonpadding_tokens": 17580, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3358652220000522, "nonpadding_tokens": 18744, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3322013329999436, "nonpadding_tokens": 2032, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3348606429999563, "nonpadding_tokens": 4960, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33244308199994066, "nonpadding_tokens": 6080, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33308449000003293, "nonpadding_tokens": 16629, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003868390000434374, "prefetch": {"data_wait_seconds": 0.014548222001238287, "consumed_batches": 1556, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2106, "tokens": 157905452, "loss_tokens": 59536, "loss": 2.437707332538296, "lr": 0.00029948945529295937, "optimizer_lrs": [0.019965963686197293, 0.00029948945529295937], "gradient_norm": 0.41560056805610657, "seconds": 0.30638977699993575, "nonpadding_tokens_per_second": 194314.577277859, "ranks": [{"compute_seconds": 0.3033260219999647, "nonpadding_tokens": 8643, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3006493390000742, "nonpadding_tokens": 3431, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3024136179999459, "nonpadding_tokens": 13552, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3031106070000078, "nonpadding_tokens": 3978, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2964638199999854, "nonpadding_tokens": 2953, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3003744339999912, "nonpadding_tokens": 5919, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2963903280000295, "nonpadding_tokens": 13832, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30380579400002716, "nonpadding_tokens": 7228, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003481939999119277, "prefetch": {"data_wait_seconds": 0.014557480001215595, "consumed_batches": 1557, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2107, "tokens": 157970724, "loss_tokens": 65272, "loss": 2.513844230680843, "lr": 0.000299488972192185, "optimizer_lrs": [0.019965931479479, 0.000299488972192185], "gradient_norm": 0.4343826472759247, "seconds": 0.2680938400000059, "nonpadding_tokens_per_second": 243466.9890214507, "ranks": [{"compute_seconds": 0.2632754549999845, "nonpadding_tokens": 4097, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26562965200002964, "nonpadding_tokens": 12784, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26297232900003564, "nonpadding_tokens": 2221, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26359688200000164, "nonpadding_tokens": 8581, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26297500700002274, "nonpadding_tokens": 12214, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26430972700006805, "nonpadding_tokens": 4969, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2622073980001005, "nonpadding_tokens": 5730, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2628897729999835, "nonpadding_tokens": 14676, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005081030000155806, "prefetch": {"data_wait_seconds": 0.014568977001161443, "consumed_batches": 1558, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2108, "tokens": 158020130, "loss_tokens": 49406, "loss": 3.0972278797109665, "lr": 0.00029948860636957455, "optimizer_lrs": [0.019965907091304972, 0.00029948860636957455], "gradient_norm": 5.83857536315918, "seconds": 0.32168114199998854, "nonpadding_tokens_per_second": 153586.8708150811, "ranks": [{"compute_seconds": 0.31823458500002744, "nonpadding_tokens": 7387, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3184867239999676, "nonpadding_tokens": 5103, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3162946589999365, "nonpadding_tokens": 4359, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3185477309999669, "nonpadding_tokens": 11718, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31517597200002, "nonpadding_tokens": 5235, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31599645799997234, "nonpadding_tokens": 7259, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31482138900003065, "nonpadding_tokens": 6028, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31923021600005086, "nonpadding_tokens": 2317, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040677199990568624, "prefetch": {"data_wait_seconds": 0.014579268001241985, "consumed_batches": 1559, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2109, "tokens": 158110296, "loss_tokens": 90166, "loss": 2.3582170662999355, "lr": 0.0002994879384061672, "optimizer_lrs": [0.01996586256041115, 0.0002994879384061672], "gradient_norm": 0.5195585489273071, "seconds": 0.25992412499999773, "nonpadding_tokens_per_second": 346893.5405668896, "ranks": [{"compute_seconds": 0.2573133209999696, "nonpadding_tokens": 6193, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25596529899996767, "nonpadding_tokens": 13418, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2552919009999641, "nonpadding_tokens": 12562, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2555121650000274, "nonpadding_tokens": 13548, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.251904725999907, "nonpadding_tokens": 7612, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25311185999999, "nonpadding_tokens": 16876, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2515511660000129, "nonpadding_tokens": 4272, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2525328689999924, "nonpadding_tokens": 15685, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037505299997064867, "prefetch": {"data_wait_seconds": 0.014588263001158452, "consumed_batches": 1560, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2110, "tokens": 158187432, "loss_tokens": 77136, "loss": 2.7617110525565236, "lr": 0.000299487366625632, "optimizer_lrs": [0.0199658244417088, 0.000299487366625632], "gradient_norm": 0.42278289794921875, "seconds": 0.3686428400000068, "nonpadding_tokens_per_second": 209243.1796586598, "ranks": [{"compute_seconds": 0.3660358009999527, "nonpadding_tokens": 18237, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3628326540000444, "nonpadding_tokens": 10998, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3652941390000706, "nonpadding_tokens": 13196, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3615686900000128, "nonpadding_tokens": 13015, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3590867650000291, "nonpadding_tokens": 5417, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3636560359999521, "nonpadding_tokens": 2028, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3660484980000547, "nonpadding_tokens": 10140, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.360161451999943, "nonpadding_tokens": 4105, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038383899993732484, "prefetch": {"data_wait_seconds": 0.014597184001175378, "consumed_batches": 1561, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2111, "tokens": 158237579, "loss_tokens": 50147, "loss": 2.3988903872614515, "lr": 0.0002994869947337276, "optimizer_lrs": [0.019965799648915174, 0.0002994869947337276], "gradient_norm": 0.40318411588668823, "seconds": 0.26485791099992184, "nonpadding_tokens_per_second": 189335.48109127386, "ranks": [{"compute_seconds": 0.2610544880000134, "nonpadding_tokens": 8950, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26240107600006013, "nonpadding_tokens": 16854, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25931421999996473, "nonpadding_tokens": 2054, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2614336330000242, "nonpadding_tokens": 5372, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2582621789999848, "nonpadding_tokens": 3720, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26051157900008093, "nonpadding_tokens": 6937, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2616027330000179, "nonpadding_tokens": 2749, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2589096320000408, "nonpadding_tokens": 3511, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004582510000545881, "prefetch": {"data_wait_seconds": 0.014608381001224302, "consumed_batches": 1562, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2112, "tokens": 158304939, "loss_tokens": 67360, "loss": 2.3671230144002373, "lr": 0.0002994864949777985, "optimizer_lrs": [0.019965766331853235, 0.0002994864949777985], "gradient_norm": 0.48527467250823975, "seconds": 0.3099979529999928, "nonpadding_tokens_per_second": 217291.75740719028, "ranks": [{"compute_seconds": 0.30713170799992895, "nonpadding_tokens": 10531, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3067381309999746, "nonpadding_tokens": 2305, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3042558160000226, "nonpadding_tokens": 11887, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3060886719999871, "nonpadding_tokens": 12314, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3074998450000521, "nonpadding_tokens": 15269, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3045226649999222, "nonpadding_tokens": 9171, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3030675979999842, "nonpadding_tokens": 4143, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3076928680000037, "nonpadding_tokens": 1740, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003729520000206321, "prefetch": {"data_wait_seconds": 0.014616270001241105, "consumed_batches": 1563, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2113, "tokens": 158372198, "loss_tokens": 67259, "loss": 1.7852417984953686, "lr": 0.00029948599572891035, "optimizer_lrs": [0.019965733048594024, 0.00029948599572891035], "gradient_norm": 0.4391499161720276, "seconds": 0.2897258259999944, "nonpadding_tokens_per_second": 232147.06444568493, "ranks": [{"compute_seconds": 0.28705873700005213, "nonpadding_tokens": 4144, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2839242009999907, "nonpadding_tokens": 10808, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28457764600000246, "nonpadding_tokens": 18346, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2873321170000054, "nonpadding_tokens": 3599, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28420657699996354, "nonpadding_tokens": 3871, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2863780069999393, "nonpadding_tokens": 10952, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28366064300007565, "nonpadding_tokens": 1759, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28480636399990544, "nonpadding_tokens": 13780, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003747420000763668, "prefetch": {"data_wait_seconds": 0.014625642001149686, "consumed_batches": 1564, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2114, "tokens": 158448181, "loss_tokens": 75983, "loss": 2.5374818627192925, "lr": 0.0002994854314324168, "optimizer_lrs": [0.019965695428827788, 0.0002994854314324168], "gradient_norm": 0.751402735710144, "seconds": 0.28911645300001965, "nonpadding_tokens_per_second": 262811.0548934925, "ranks": [{"compute_seconds": 0.2852323370000249, "nonpadding_tokens": 6365, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28647021000006134, "nonpadding_tokens": 7951, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28543108399992434, "nonpadding_tokens": 3404, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28395148300000983, "nonpadding_tokens": 20094, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28682725499993467, "nonpadding_tokens": 6659, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2840717849999237, "nonpadding_tokens": 17504, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28272888000003604, "nonpadding_tokens": 2998, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28672476799999913, "nonpadding_tokens": 11008, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003489329999410984, "prefetch": {"data_wait_seconds": 0.014634617001050287, "consumed_batches": 1565, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2115, "tokens": 158500504, "loss_tokens": 52323, "loss": 2.2178117176002905, "lr": 0.00029948504266998657, "optimizer_lrs": [0.01996566951133244, 0.00029948504266998657], "gradient_norm": 0.7199718356132507, "seconds": 0.29746856500003105, "nonpadding_tokens_per_second": 175894.21591486328, "ranks": [{"compute_seconds": 0.2935528749999321, "nonpadding_tokens": 2693, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29483088999995744, "nonpadding_tokens": 6829, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29150784700004806, "nonpadding_tokens": 4408, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.291983651999999, "nonpadding_tokens": 5298, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2951358909999726, "nonpadding_tokens": 7541, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2921347239999932, "nonpadding_tokens": 11241, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29078017500000897, "nonpadding_tokens": 10293, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29505137900002865, "nonpadding_tokens": 4020, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036203899992415245, "prefetch": {"data_wait_seconds": 0.01464349100110951, "consumed_batches": 1566, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2116, "tokens": 158568613, "loss_tokens": 68109, "loss": 2.626217028953589, "lr": 0.00029948453639731153, "optimizer_lrs": [0.01996563575982077, 0.00029948453639731153], "gradient_norm": 0.4372253119945526, "seconds": 0.2709077580000212, "nonpadding_tokens_per_second": 251410.29737507433, "ranks": [{"compute_seconds": 0.26568470699999125, "nonpadding_tokens": 3616, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26567736499998773, "nonpadding_tokens": 7975, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26856495699996685, "nonpadding_tokens": 8725, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2650018030000183, "nonpadding_tokens": 7136, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2630880290000732, "nonpadding_tokens": 3648, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2661329810000552, "nonpadding_tokens": 3732, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2636024199999838, "nonpadding_tokens": 14491, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26408820699998614, "nonpadding_tokens": 18786, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034535100007815345, "prefetch": {"data_wait_seconds": 0.014651568001070814, "consumed_batches": 1567, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2117, "tokens": 158640565, "loss_tokens": 71952, "loss": 2.4553293775711587, "lr": 0.00029948400128889817, "optimizer_lrs": [0.019965600085926547, 0.00029948400128889817], "gradient_norm": 0.3876151740550995, "seconds": 0.2839355969999815, "nonpadding_tokens_per_second": 253409.5786517556, "ranks": [{"compute_seconds": 0.2794507830000157, "nonpadding_tokens": 4865, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2811964429999989, "nonpadding_tokens": 7334, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2780701939999517, "nonpadding_tokens": 15589, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27910957500000677, "nonpadding_tokens": 10320, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2815713190000224, "nonpadding_tokens": 12879, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27984898099998645, "nonpadding_tokens": 13222, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27684111500002473, "nonpadding_tokens": 2145, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2773529959999905, "nonpadding_tokens": 5598, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035661899994465784, "prefetch": {"data_wait_seconds": 0.014660318001077144, "consumed_batches": 1568, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2118, "tokens": 158716290, "loss_tokens": 75725, "loss": 2.377948374050842, "lr": 0.00029948343782136116, "optimizer_lrs": [0.01996556252142408, 0.00029948343782136116], "gradient_norm": 0.46645689010620117, "seconds": 0.2998117509999929, "nonpadding_tokens_per_second": 252575.15673560704, "ranks": [{"compute_seconds": 0.29590955799994845, "nonpadding_tokens": 3646, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29686264999998, "nonpadding_tokens": 2153, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2932317019999573, "nonpadding_tokens": 10518, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2961001370000531, "nonpadding_tokens": 4338, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29284196300000076, "nonpadding_tokens": 14162, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29541689399991355, "nonpadding_tokens": 6886, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.292296427999986, "nonpadding_tokens": 16041, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2974551500000189, "nonpadding_tokens": 17981, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039144099991972325, "prefetch": {"data_wait_seconds": 0.014668373001043733, "consumed_batches": 1569, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2119, "tokens": 158791307, "loss_tokens": 75017, "loss": 2.444808343442153, "lr": 0.00029948287931942826, "optimizer_lrs": [0.019965525287961885, 0.00029948287931942826], "gradient_norm": 0.44816747307777405, "seconds": 0.3249056260000316, "nonpadding_tokens_per_second": 230888.58424382194, "ranks": [{"compute_seconds": 0.3223861179999403, "nonpadding_tokens": 9204, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3223010719999593, "nonpadding_tokens": 3497, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3215821329999926, "nonpadding_tokens": 4022, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3200737390000086, "nonpadding_tokens": 18936, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3226093130000436, "nonpadding_tokens": 16837, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32153104499991514, "nonpadding_tokens": 7933, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3185249449999219, "nonpadding_tokens": 3544, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31975353699999687, "nonpadding_tokens": 11044, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039702499998384155, "prefetch": {"data_wait_seconds": 0.014676961001057975, "consumed_batches": 1570, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2120, "tokens": 158861743, "loss_tokens": 70436, "loss": 2.399889616105401, "lr": 0.0002994823546489083, "optimizer_lrs": [0.01996549030992722, 0.0002994823546489083], "gradient_norm": 0.636934220790863, "seconds": 0.2893450470000971, "nonpadding_tokens_per_second": 243432.5409412533, "ranks": [{"compute_seconds": 0.252248780000059, "nonpadding_tokens": 7952, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26171565900006044, "nonpadding_tokens": 8887, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2639963000000307, "nonpadding_tokens": 9894, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26285212699997373, "nonpadding_tokens": 15212, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2611898459999793, "nonpadding_tokens": 5089, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26240051100000983, "nonpadding_tokens": 7256, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2611235350000243, "nonpadding_tokens": 10471, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26181400299992674, "nonpadding_tokens": 5675, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.001771811999901729, "prefetch": {"data_wait_seconds": 0.01468628100106173, "consumed_batches": 1571, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2121, "tokens": 158931431, "loss_tokens": 69688, "loss": 2.2616851179543107, "lr": 0.0002994818352888589, "optimizer_lrs": [0.01996545568592393, 0.0002994818352888589], "gradient_norm": 0.3737637400627136, "seconds": 0.347840384000051, "nonpadding_tokens_per_second": 200344.76502874887, "ranks": [{"compute_seconds": 0.34268984599998475, "nonpadding_tokens": 11017, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34099968200007424, "nonpadding_tokens": 15483, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34166684399997393, "nonpadding_tokens": 17637, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3449794089999614, "nonpadding_tokens": 7025, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3452463759999773, "nonpadding_tokens": 2635, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3433321429999978, "nonpadding_tokens": 9738, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3408661469999288, "nonpadding_tokens": 3638, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34552453899993907, "nonpadding_tokens": 2515, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035963499999525084, "prefetch": {"data_wait_seconds": 0.01469533500103637, "consumed_batches": 1572, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2122, "tokens": 159003962, "loss_tokens": 72531, "loss": 2.652701301857137, "lr": 0.00029948129446490487, "optimizer_lrs": [0.019965419630993658, 0.00029948129446490487], "gradient_norm": 0.48455610871315, "seconds": 0.26856136000003517, "nonpadding_tokens_per_second": 270072.35888286575, "ranks": [{"compute_seconds": 0.26624715499997365, "nonpadding_tokens": 8780, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2648572329999297, "nonpadding_tokens": 17521, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26426627399996505, "nonpadding_tokens": 9676, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2628718339999523, "nonpadding_tokens": 19436, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26545939300001464, "nonpadding_tokens": 1915, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2638789769999903, "nonpadding_tokens": 8094, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2614164950000486, "nonpadding_tokens": 2024, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2623125819999359, "nonpadding_tokens": 5085, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000356038999939301, "prefetch": {"data_wait_seconds": 0.014704244001066513, "consumed_batches": 1573, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2123, "tokens": 159054783, "loss_tokens": 50821, "loss": 2.3105470671572776, "lr": 0.00029948091535278394, "optimizer_lrs": [0.019965394356852264, 0.00029948091535278394], "gradient_norm": 0.37601590156555176, "seconds": 0.26315720699994927, "nonpadding_tokens_per_second": 193120.30470064154, "ranks": [{"compute_seconds": 0.2250035829999888, "nonpadding_tokens": 4462, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24715473300000212, "nonpadding_tokens": 4986, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24693756900001063, "nonpadding_tokens": 1898, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.247208652999916, "nonpadding_tokens": 10357, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24580311599993365, "nonpadding_tokens": 3371, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24866389299995717, "nonpadding_tokens": 5673, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25002047700002095, "nonpadding_tokens": 7779, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2508075269998926, "nonpadding_tokens": 12295, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.001876088000017262, "prefetch": {"data_wait_seconds": 0.014712952001062263, "consumed_batches": 1574, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2124, "tokens": 159131183, "loss_tokens": 76400, "loss": 2.5983409685863874, "lr": 0.00029948034516754415, "optimizer_lrs": [0.019965356344502944, 0.00029948034516754415], "gradient_norm": 0.35191240906715393, "seconds": 0.274961843999904, "nonpadding_tokens_per_second": 277856.7341875503, "ranks": [{"compute_seconds": 0.2726113800000576, "nonpadding_tokens": 5642, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26891572099998484, "nonpadding_tokens": 8624, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2719327060000296, "nonpadding_tokens": 15499, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2699058009999362, "nonpadding_tokens": 7269, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2687323309999101, "nonpadding_tokens": 15088, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.271331590999921, "nonpadding_tokens": 3589, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2686415360000183, "nonpadding_tokens": 9428, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2692624920000526, "nonpadding_tokens": 11261, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038432899998497305, "prefetch": {"data_wait_seconds": 0.014722398001097758, "consumed_batches": 1575, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2125, "tokens": 159224901, "loss_tokens": 93718, "loss": 2.3211362811839775, "lr": 0.0002994796453087194, "optimizer_lrs": [0.01996530968724796, 0.0002994796453087194], "gradient_norm": 0.4433036148548126, "seconds": 0.28136750799990295, "nonpadding_tokens_per_second": 333080.3924951858, "ranks": [{"compute_seconds": 0.277054851999992, "nonpadding_tokens": 14427, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2749979259999691, "nonpadding_tokens": 5296, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2771532980000302, "nonpadding_tokens": 9969, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27785742099990784, "nonpadding_tokens": 13421, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2785443549998945, "nonpadding_tokens": 6870, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2754048199999488, "nonpadding_tokens": 12333, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2737343599999349, "nonpadding_tokens": 10796, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27890556600004857, "nonpadding_tokens": 20606, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003675709999697574, "prefetch": {"data_wait_seconds": 0.01473082100115164, "consumed_batches": 1576, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2126, "tokens": 159275246, "loss_tokens": 50345, "loss": 2.6440510477703842, "lr": 0.00029947926915279257, "optimizer_lrs": [0.019965284610186172, 0.00029947926915279257], "gradient_norm": 0.43548500537872314, "seconds": 0.298041501000057, "nonpadding_tokens_per_second": 168919.42843889506, "ranks": [{"compute_seconds": 0.29277265700000044, "nonpadding_tokens": 9543, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2957043960000192, "nonpadding_tokens": 3965, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29411489400001756, "nonpadding_tokens": 2264, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29475587799993264, "nonpadding_tokens": 1276, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29116494100003365, "nonpadding_tokens": 2012, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.292458742000008, "nonpadding_tokens": 11939, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29110560299989174, "nonpadding_tokens": 13101, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2918389850000267, "nonpadding_tokens": 6245, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036239099995327706, "prefetch": {"data_wait_seconds": 0.014738931001218702, "consumed_batches": 1577, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2127, "tokens": 159346409, "loss_tokens": 71163, "loss": 2.3220715557944436, "lr": 0.0002994787372224512, "optimizer_lrs": [0.019965249148163416, 0.0002994787372224512], "gradient_norm": 0.4116840064525604, "seconds": 0.27879577999999583, "nonpadding_tokens_per_second": 255251.35280025066, "ranks": [{"compute_seconds": 0.2763944910000191, "nonpadding_tokens": 5270, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2759041059999845, "nonpadding_tokens": 13776, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2755201020000868, "nonpadding_tokens": 8527, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27553318999991916, "nonpadding_tokens": 11847, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2726155509999444, "nonpadding_tokens": 12879, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2734936669999115, "nonpadding_tokens": 9332, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2719671479999306, "nonpadding_tokens": 2389, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27650692400004573, "nonpadding_tokens": 7143, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035948900006133044, "prefetch": {"data_wait_seconds": 0.01474802000127795, "consumed_batches": 1578, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2128, "tokens": 159409565, "loss_tokens": 63156, "loss": 2.8745046986826273, "lr": 0.00029947826491596747, "optimizer_lrs": [0.0199652176610645, 0.00029947826491596747], "gradient_norm": 0.47307083010673523, "seconds": 0.2693739049999522, "nonpadding_tokens_per_second": 234454.78135683265, "ranks": [{"compute_seconds": 0.2670158219999621, "nonpadding_tokens": 7325, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2633289820000755, "nonpadding_tokens": 1497, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26627705499993226, "nonpadding_tokens": 3415, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26466034500003843, "nonpadding_tokens": 14505, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26316432500004794, "nonpadding_tokens": 2581, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26412917400000424, "nonpadding_tokens": 24871, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.262830364000024, "nonpadding_tokens": 4442, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26403396099999554, "nonpadding_tokens": 4520, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035448299991003296, "prefetch": {"data_wait_seconds": 0.014755270001344911, "consumed_batches": 1579, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2129, "tokens": 159456714, "loss_tokens": 47149, "loss": 2.3752528553097627, "lr": 0.00029947791217723276, "optimizer_lrs": [0.019965194145148853, 0.00029947791217723276], "gradient_norm": 0.8036071062088013, "seconds": 0.26540600299995276, "nonpadding_tokens_per_second": 177648.58167133617, "ranks": [{"compute_seconds": 0.26290560299992194, "nonpadding_tokens": 5710, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2626886859999331, "nonpadding_tokens": 5734, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2625146870000208, "nonpadding_tokens": 2323, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25865111299992805, "nonpadding_tokens": 3467, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2573019490000661, "nonpadding_tokens": 6687, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2604760939999551, "nonpadding_tokens": 12280, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.263008066999987, "nonpadding_tokens": 2712, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2580359219999764, "nonpadding_tokens": 8236, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039955500005817157, "prefetch": {"data_wait_seconds": 0.014779441001294913, "consumed_batches": 1580, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2130, "tokens": 159506848, "loss_tokens": 50134, "loss": 2.297165004039175, "lr": 0.0002994775369761264, "optimizer_lrs": [0.01996516913174176, 0.0002994775369761264], "gradient_norm": 0.4480843245983124, "seconds": 0.34439239899995755, "nonpadding_tokens_per_second": 145572.3185110313, "ranks": [{"compute_seconds": 0.34107294399996135, "nonpadding_tokens": 4169, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33947580699998525, "nonpadding_tokens": 4107, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3406420239999761, "nonpadding_tokens": 4507, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.336897338999961, "nonpadding_tokens": 13509, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3419555420000506, "nonpadding_tokens": 6093, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3393510859999651, "nonpadding_tokens": 4815, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3350871579999648, "nonpadding_tokens": 5107, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3356608890001098, "nonpadding_tokens": 7827, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042715799997949944, "prefetch": {"data_wait_seconds": 0.01478929600132517, "consumed_batches": 1581, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2131, "tokens": 159556400, "loss_tokens": 49552, "loss": 2.230700356443736, "lr": 0.0002994771659985107, "optimizer_lrs": [0.019965144399900714, 0.0002994771659985107], "gradient_norm": 0.37723788619041443, "seconds": 0.2371687090000023, "nonpadding_tokens_per_second": 208931.44044562607, "ranks": [{"compute_seconds": 0.23333097900001576, "nonpadding_tokens": 9914, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23163267699999324, "nonpadding_tokens": 6008, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23460784199994578, "nonpadding_tokens": 7780, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2305110979999654, "nonpadding_tokens": 4980, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2285275230000252, "nonpadding_tokens": 3289, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2315636999999242, "nonpadding_tokens": 5863, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23487093099993217, "nonpadding_tokens": 4082, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22957031699991148, "nonpadding_tokens": 7636, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003576159999738593, "prefetch": {"data_wait_seconds": 0.01479736300127854, "consumed_batches": 1582, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2132, "tokens": 159607291, "loss_tokens": 50891, "loss": 2.545473260252304, "lr": 0.0002994767848595147, "optimizer_lrs": [0.019965118990634315, 0.0002994767848595147], "gradient_norm": 0.5187166929244995, "seconds": 0.30901572499999475, "nonpadding_tokens_per_second": 164687.4119431976, "ranks": [{"compute_seconds": 0.3063165179999032, "nonpadding_tokens": 1004, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30420205900009023, "nonpadding_tokens": 10178, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30645930100001806, "nonpadding_tokens": 9709, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3017492819999461, "nonpadding_tokens": 5369, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3003449840000485, "nonpadding_tokens": 6856, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30417639500001314, "nonpadding_tokens": 2640, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3001528130000679, "nonpadding_tokens": 3602, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3010272609999447, "nonpadding_tokens": 11533, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003462629999830824, "prefetch": {"data_wait_seconds": 0.01480609900131924, "consumed_batches": 1583, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2133, "tokens": 159672124, "loss_tokens": 64833, "loss": 2.425592965773603, "lr": 0.00029947629910365576, "optimizer_lrs": [0.019965086606910386, 0.00029947629910365576], "gradient_norm": 0.4854210913181305, "seconds": 0.29060565699990093, "nonpadding_tokens_per_second": 223096.13883401486, "ranks": [{"compute_seconds": 0.28789826900003845, "nonpadding_tokens": 3304, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2853448019999405, "nonpadding_tokens": 10146, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2873651880000807, "nonpadding_tokens": 6808, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2827103969999598, "nonpadding_tokens": 17134, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28115093500002786, "nonpadding_tokens": 5009, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2841237559999854, "nonpadding_tokens": 9078, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2881069180000395, "nonpadding_tokens": 5683, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2817481960000805, "nonpadding_tokens": 7671, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003555670000423561, "prefetch": {"data_wait_seconds": 0.01481369500136509, "consumed_batches": 1584, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2134, "tokens": 159778228, "loss_tokens": 106104, "loss": 2.360484760235241, "lr": 0.000299475503642947, "optimizer_lrs": [0.01996503357619647, 0.000299475503642947], "gradient_norm": 0.3773154020309448, "seconds": 0.37609697499999584, "nonpadding_tokens_per_second": 282118.72749043297, "ranks": [{"compute_seconds": 0.372438150999983, "nonpadding_tokens": 26707, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37126989699993374, "nonpadding_tokens": 10385, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3680409099999906, "nonpadding_tokens": 2655, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.36865418800005045, "nonpadding_tokens": 14208, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36683891099994526, "nonpadding_tokens": 6492, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37027909000005366, "nonpadding_tokens": 5989, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36721799300005387, "nonpadding_tokens": 28662, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3735875050000459, "nonpadding_tokens": 11006, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042354200002137077, "prefetch": {"data_wait_seconds": 0.014823185001390016, "consumed_batches": 1585, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2135, "tokens": 159835066, "loss_tokens": 56838, "loss": 2.612371454836553, "lr": 0.0002994750772811712, "optimizer_lrs": [0.019965005152078084, 0.0002994750772811712], "gradient_norm": 0.43806344270706177, "seconds": 0.3061547159999236, "nonpadding_tokens_per_second": 185651.23132061824, "ranks": [{"compute_seconds": 0.3016638140001078, "nonpadding_tokens": 10520, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2996413319999647, "nonpadding_tokens": 3045, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2978223869999965, "nonpadding_tokens": 12243, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2980955289999656, "nonpadding_tokens": 3137, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30358347900005356, "nonpadding_tokens": 14184, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29898210800001834, "nonpadding_tokens": 3318, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30289976400001706, "nonpadding_tokens": 4057, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30353995199993733, "nonpadding_tokens": 6334, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004479180000771521, "prefetch": {"data_wait_seconds": 0.014833317001333057, "consumed_batches": 1586, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2136, "tokens": 159944109, "loss_tokens": 109043, "loss": 2.1855271544253183, "lr": 0.0002994742588273761, "optimizer_lrs": [0.01996495058849174, 0.0002994742588273761], "gradient_norm": 1.2033406496047974, "seconds": 0.3259215819999781, "nonpadding_tokens_per_second": 334568.2091099058, "ranks": [{"compute_seconds": 0.32154003100004047, "nonpadding_tokens": 14838, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31880678799996076, "nonpadding_tokens": 6621, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31669615600003453, "nonpadding_tokens": 15021, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3219879079999828, "nonpadding_tokens": 13705, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32241397799998595, "nonpadding_tokens": 9474, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3197686429999749, "nonpadding_tokens": 17507, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3159585559999414, "nonpadding_tokens": 20195, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.323356710999974, "nonpadding_tokens": 11682, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039033799998833274, "prefetch": {"data_wait_seconds": 0.014842939001326272, "consumed_batches": 1587, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2137, "tokens": 159999121, "loss_tokens": 55012, "loss": 2.5009614379589906, "lr": 0.0002994738456773997, "optimizer_lrs": [0.019964923045159982, 0.0002994738456773997], "gradient_norm": 0.4718952775001526, "seconds": 0.2481251169999723, "nonpadding_tokens_per_second": 221710.7266895763, "ranks": [{"compute_seconds": 0.24149713099996006, "nonpadding_tokens": 7291, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2423573360000546, "nonpadding_tokens": 6046, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2392743130000099, "nonpadding_tokens": 6853, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23958612800004175, "nonpadding_tokens": 9826, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24557598499995947, "nonpadding_tokens": 3742, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24235061400008817, "nonpadding_tokens": 9256, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24518839000006665, "nonpadding_tokens": 6611, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2393402550000019, "nonpadding_tokens": 5387, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004057250000641943, "prefetch": {"data_wait_seconds": 0.014852974001271946, "consumed_batches": 1588, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2138, "tokens": 160055783, "loss_tokens": 56662, "loss": 2.3327381556422293, "lr": 0.00029947341996631376, "optimizer_lrs": [0.019964894664420918, 0.00029947341996631376], "gradient_norm": 0.4318937063217163, "seconds": 0.42965273400000115, "nonpadding_tokens_per_second": 131878.59756526034, "ranks": [{"compute_seconds": 0.42622251099999175, "nonpadding_tokens": 5484, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4256667719999996, "nonpadding_tokens": 4452, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.42590800800007855, "nonpadding_tokens": 5286, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.42650732499998867, "nonpadding_tokens": 11405, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.42198330999997324, "nonpadding_tokens": 8554, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.42365795000000617, "nonpadding_tokens": 11152, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4216837829999349, "nonpadding_tokens": 8396, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.42714742399994066, "nonpadding_tokens": 1933, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004579120000016701, "prefetch": {"data_wait_seconds": 0.014863689001302305, "consumed_batches": 1589, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2139, "tokens": 160088920, "loss_tokens": 33137, "loss": 2.478914949678607, "lr": 0.00029947317092286676, "optimizer_lrs": [0.01996487806152445, 0.00029947317092286676], "gradient_norm": 0.47506073117256165, "seconds": 0.27824772100007067, "nonpadding_tokens_per_second": 119091.72115013149, "ranks": [{"compute_seconds": 0.2474004949999653, "nonpadding_tokens": 2865, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26954503599995405, "nonpadding_tokens": 6101, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26708573600001273, "nonpadding_tokens": 2445, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2725730710000107, "nonpadding_tokens": 5679, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27289063399996394, "nonpadding_tokens": 9242, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2708550849999938, "nonpadding_tokens": 1829, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2726720790000172, "nonpadding_tokens": 1801, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27387988999998925, "nonpadding_tokens": 3175, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004216549999682684, "prefetch": {"data_wait_seconds": 0.014874548001287735, "consumed_batches": 1590, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2140, "tokens": 160159425, "loss_tokens": 70505, "loss": 2.423461766895965, "lr": 0.00029947264084207077, "optimizer_lrs": [0.01996484272280472, 0.00029947264084207077], "gradient_norm": 0.42033880949020386, "seconds": 0.29736826900000324, "nonpadding_tokens_per_second": 237096.58141097505, "ranks": [{"compute_seconds": 0.2411112879999564, "nonpadding_tokens": 7746, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.239971905999937, "nonpadding_tokens": 7995, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24268979700002546, "nonpadding_tokens": 8928, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23726046299998416, "nonpadding_tokens": 4556, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23587950900002852, "nonpadding_tokens": 18664, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23891111699992962, "nonpadding_tokens": 6909, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23599397800001043, "nonpadding_tokens": 10741, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23675396300006923, "nonpadding_tokens": 4966, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.033686476999946535, "prefetch": {"data_wait_seconds": 0.014883082001304615, "consumed_batches": 1591, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2141, "tokens": 160211332, "loss_tokens": 51907, "loss": 2.6868103651723274, "lr": 0.00029947225041740584, "optimizer_lrs": [0.019964816694493723, 0.00029947225041740584], "gradient_norm": 0.45780226588249207, "seconds": 0.28094860700002755, "nonpadding_tokens_per_second": 184756.2105904832, "ranks": [{"compute_seconds": 0.27865811699996357, "nonpadding_tokens": 5337, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2745190210000601, "nonpadding_tokens": 8861, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2719038589999627, "nonpadding_tokens": 5473, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27212712900006863, "nonpadding_tokens": 2755, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27091822900001716, "nonpadding_tokens": 5037, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2753219120000949, "nonpadding_tokens": 2874, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27093170900002406, "nonpadding_tokens": 10564, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2718422589999818, "nonpadding_tokens": 11006, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038893999999345397, "prefetch": {"data_wait_seconds": 0.014891760001276566, "consumed_batches": 1592, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2142, "tokens": 160313208, "loss_tokens": 101876, "loss": 2.339211700989438, "lr": 0.00029947148372577097, "optimizer_lrs": [0.019964765581718065, 0.00029947148372577097], "gradient_norm": 0.5010887980461121, "seconds": 0.25329285600003004, "nonpadding_tokens_per_second": 402206.3693734336, "ranks": [{"compute_seconds": 0.2470586680000224, "nonpadding_tokens": 7509, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24652594999997746, "nonpadding_tokens": 10922, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24402479399998356, "nonpadding_tokens": 18328, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24471599900005003, "nonpadding_tokens": 16276, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2500540790000514, "nonpadding_tokens": 24726, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2475261849999697, "nonpadding_tokens": 14773, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24974718500004656, "nonpadding_tokens": 6057, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2507849050000459, "nonpadding_tokens": 3285, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000366734999943219, "prefetch": {"data_wait_seconds": 0.014902355001211731, "consumed_batches": 1593, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2143, "tokens": 160400180, "loss_tokens": 86972, "loss": 2.511448548095939, "lr": 0.0002994708287581954, "optimizer_lrs": [0.01996472191721303, 0.0002994708287581954], "gradient_norm": 0.3738742768764496, "seconds": 0.2542250960000274, "nonpadding_tokens_per_second": 342106.2726238114, "ranks": [{"compute_seconds": 0.25180861299998014, "nonpadding_tokens": 9562, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.249066675999984, "nonpadding_tokens": 20331, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24596571699999004, "nonpadding_tokens": 15050, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24649509999994734, "nonpadding_tokens": 6850, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24468255299996144, "nonpadding_tokens": 12022, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2478971120000324, "nonpadding_tokens": 16519, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24508528800004115, "nonpadding_tokens": 3892, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24614111400001093, "nonpadding_tokens": 2746, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039608299994142726, "prefetch": {"data_wait_seconds": 0.014910844001249757, "consumed_batches": 1594, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2144, "tokens": 160473744, "loss_tokens": 73564, "loss": 2.7571721137376977, "lr": 0.0002994702744474144, "optimizer_lrs": [0.019964684963160964, 0.0002994702744474144], "gradient_norm": 0.410915732383728, "seconds": 0.30154825700003585, "nonpadding_tokens_per_second": 243954.32005428986, "ranks": [{"compute_seconds": 0.2988390419999405, "nonpadding_tokens": 6394, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.294845075000012, "nonpadding_tokens": 8687, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2975785169999199, "nonpadding_tokens": 6226, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2982120980000218, "nonpadding_tokens": 9685, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2985209710000163, "nonpadding_tokens": 22422, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2940184319999162, "nonpadding_tokens": 3457, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2911101500000086, "nonpadding_tokens": 3893, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2915751929999715, "nonpadding_tokens": 12800, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004167220000681482, "prefetch": {"data_wait_seconds": 0.0149202960012218, "consumed_batches": 1595, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2145, "tokens": 160548125, "loss_tokens": 74381, "loss": 2.6038996013766957, "lr": 0.0002994697136860294, "optimizer_lrs": [0.01996464757906863, 0.0002994697136860294], "gradient_norm": 0.4370433986186981, "seconds": 0.3006032739999682, "nonpadding_tokens_per_second": 247439.08810523423, "ranks": [{"compute_seconds": 0.2960308359999999, "nonpadding_tokens": 6852, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29451199400000405, "nonpadding_tokens": 17528, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29198538699995424, "nonpadding_tokens": 6751, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29743520499994247, "nonpadding_tokens": 9909, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29769017399996756, "nonpadding_tokens": 8144, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2931305600000087, "nonpadding_tokens": 9972, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29067683299990676, "nonpadding_tokens": 6247, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2980008330000601, "nonpadding_tokens": 8978, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000495036999950571, "prefetch": {"data_wait_seconds": 0.014932238001236442, "consumed_batches": 1596, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2146, "tokens": 160624227, "loss_tokens": 76102, "loss": 2.747508894312896, "lr": 0.000299469139643534, "optimizer_lrs": [0.019964609309568934, 0.000299469139643534], "gradient_norm": 0.4627869129180908, "seconds": 0.3678492249999863, "nonpadding_tokens_per_second": 206883.67632146794, "ranks": [{"compute_seconds": 0.35772083400001975, "nonpadding_tokens": 13147, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36188422999998693, "nonpadding_tokens": 2525, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3642030439999644, "nonpadding_tokens": 6311, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35744479799996043, "nonpadding_tokens": 11070, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36523263600008704, "nonpadding_tokens": 2650, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35840533199996116, "nonpadding_tokens": 2603, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35623764800004665, "nonpadding_tokens": 16458, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3564701120000109, "nonpadding_tokens": 21338, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004233739999790487, "prefetch": {"data_wait_seconds": 0.014942555001269398, "consumed_batches": 1597, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2147, "tokens": 160668713, "loss_tokens": 44486, "loss": 2.3692153500539495, "lr": 0.0002994688039391131, "optimizer_lrs": [0.019964586929274206, 0.0002994688039391131], "gradient_norm": 0.43928101658821106, "seconds": 0.2867302089999839, "nonpadding_tokens_per_second": 155149.3306378558, "ranks": [{"compute_seconds": 0.27893517699999393, "nonpadding_tokens": 3762, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27950045000000046, "nonpadding_tokens": 7427, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28218430200001876, "nonpadding_tokens": 3034, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27714220199993633, "nonpadding_tokens": 6375, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28309210600002643, "nonpadding_tokens": 4056, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28067608500009555, "nonpadding_tokens": 5049, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2840686160000132, "nonpadding_tokens": 9749, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2837181599999212, "nonpadding_tokens": 5034, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003989519999549884, "prefetch": {"data_wait_seconds": 0.014951896001278442, "consumed_batches": 1598, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2148, "tokens": 160728936, "loss_tokens": 60223, "loss": 2.502835295485114, "lr": 0.00029946834930994365, "optimizer_lrs": [0.01996455662066291, 0.00029946834930994365], "gradient_norm": 0.3554568886756897, "seconds": 0.29882264800005487, "nonpadding_tokens_per_second": 201534.25586399643, "ranks": [{"compute_seconds": 0.2935484010000664, "nonpadding_tokens": 5838, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2904316789999939, "nonpadding_tokens": 2874, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2869742560000077, "nonpadding_tokens": 11068, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29371878100005233, "nonpadding_tokens": 4216, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.286076372000025, "nonpadding_tokens": 9825, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2894443569999794, "nonpadding_tokens": 2049, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29382854499999667, "nonpadding_tokens": 15946, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29596914400008245, "nonpadding_tokens": 8407, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041977400007908727, "prefetch": {"data_wait_seconds": 0.01496100800132183, "consumed_batches": 1599, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2149, "tokens": 160787222, "loss_tokens": 58286, "loss": 2.2759094164979583, "lr": 0.0002994679091185529, "optimizer_lrs": [0.019964527274570192, 0.0002994679091185529], "gradient_norm": 0.4333532452583313, "seconds": 0.24648081699990598, "nonpadding_tokens_per_second": 236472.76372027863, "ranks": [{"compute_seconds": 0.2430444090000492, "nonpadding_tokens": 7795, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2397845770000231, "nonpadding_tokens": 1346, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2366614730000265, "nonpadding_tokens": 9402, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24357468700009122, "nonpadding_tokens": 5615, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2359920299999203, "nonpadding_tokens": 5750, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24146013300003233, "nonpadding_tokens": 6812, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2360643600000003, "nonpadding_tokens": 13835, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2366335759999174, "nonpadding_tokens": 7731, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042950400006702694, "prefetch": {"data_wait_seconds": 0.014972572001354933, "consumed_batches": 1600, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2150, "tokens": 160842252, "loss_tokens": 55030, "loss": 2.3427586373341813, "lr": 0.00029946749335049315, "optimizer_lrs": [0.019964499556699544, 0.00029946749335049315], "gradient_norm": 0.36473602056503296, "seconds": 0.32186828199996853, "nonpadding_tokens_per_second": 170970.55869582508, "ranks": [{"compute_seconds": 0.31924726800002645, "nonpadding_tokens": 9646, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3159976319999487, "nonpadding_tokens": 10333, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3132827009999346, "nonpadding_tokens": 6117, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31384908199993333, "nonpadding_tokens": 1619, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3192791760000091, "nonpadding_tokens": 2951, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31660750599996845, "nonpadding_tokens": 9970, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31244859299999916, "nonpadding_tokens": 3170, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3129986829999325, "nonpadding_tokens": 11224, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047997800004395685, "prefetch": {"data_wait_seconds": 0.01498363900130073, "consumed_batches": 1601, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2151, "tokens": 160911973, "loss_tokens": 69721, "loss": 2.475465381664061, "lr": 0.0002994669663548266, "optimizer_lrs": [0.01996446442365511, 0.0002994669663548266], "gradient_norm": 0.3895840346813202, "seconds": 0.35184882099997594, "nonpadding_tokens_per_second": 198156.12796953146, "ranks": [{"compute_seconds": 0.34944295200000397, "nonpadding_tokens": 8631, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3452537429999438, "nonpadding_tokens": 5974, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34817794299999605, "nonpadding_tokens": 9959, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3437238230000048, "nonpadding_tokens": 4948, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34229810700003327, "nonpadding_tokens": 19807, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34874350099994444, "nonpadding_tokens": 4155, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34887477500001296, "nonpadding_tokens": 12164, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3426656210000374, "nonpadding_tokens": 4083, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039624500004720176, "prefetch": {"data_wait_seconds": 0.014993418001225223, "consumed_batches": 1602, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2152, "tokens": 160969818, "loss_tokens": 57845, "loss": 2.334704274353877, "lr": 0.00029946652892808554, "optimizer_lrs": [0.01996443526187237, 0.00029946652892808554], "gradient_norm": 0.5461835265159607, "seconds": 0.3261879209999279, "nonpadding_tokens_per_second": 177336.42564898284, "ranks": [{"compute_seconds": 0.323615694999944, "nonpadding_tokens": 6123, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3185313110000152, "nonpadding_tokens": 3452, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3219215860000304, "nonpadding_tokens": 5657, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32374674699997286, "nonpadding_tokens": 8790, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3232482060000166, "nonpadding_tokens": 5435, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3204885779999813, "nonpadding_tokens": 6494, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3149896050000507, "nonpadding_tokens": 6857, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3167728260000331, "nonpadding_tokens": 15037, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038455900005374133, "prefetch": {"data_wait_seconds": 0.015002693001250123, "consumed_batches": 1603, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2153, "tokens": 161022003, "loss_tokens": 52185, "loss": 2.796436356232634, "lr": 0.00029946613414891135, "optimizer_lrs": [0.019964408943260757, 0.00029946613414891135], "gradient_norm": 0.39092597365379333, "seconds": 0.3609118800000033, "nonpadding_tokens_per_second": 144592.08159066286, "ranks": [{"compute_seconds": 0.3565648040000724, "nonpadding_tokens": 9316, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3533737029999884, "nonpadding_tokens": 5517, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35810140699993553, "nonpadding_tokens": 5525, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3566396789999544, "nonpadding_tokens": 5158, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3504692479999676, "nonpadding_tokens": 3704, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3531585380000024, "nonpadding_tokens": 5801, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3566024250000055, "nonpadding_tokens": 1773, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.357629832999919, "nonpadding_tokens": 15391, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043736399993576924, "prefetch": {"data_wait_seconds": 0.015013569001283145, "consumed_batches": 1604, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2154, "tokens": 161084807, "loss_tokens": 62804, "loss": 2.521211328100121, "lr": 0.00029946565884384673, "optimizer_lrs": [0.01996437725625645, 0.00029946565884384673], "gradient_norm": 0.41830605268478394, "seconds": 0.6886745159999919, "nonpadding_tokens_per_second": 91195.4755706406, "ranks": [{"compute_seconds": 0.6803399350000063, "nonpadding_tokens": 5487, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.6809325780000108, "nonpadding_tokens": 2625, "peak_allocated_bytes": 14990164480, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.6782083850000618, "nonpadding_tokens": 10002, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.6848334410000234, "nonpadding_tokens": 8151, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.686058168000045, "nonpadding_tokens": 2727, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.6804419140000846, "nonpadding_tokens": 11352, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.6774570960000119, "nonpadding_tokens": 17142, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.6775478670000439, "nonpadding_tokens": 5318, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040781500001685345, "prefetch": {"data_wait_seconds": 0.015023393001300178, "consumed_batches": 1605, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2155, "tokens": 161157809, "loss_tokens": 73002, "loss": 2.3765575343826195, "lr": 0.0002994651060943421, "optimizer_lrs": [0.019964340406289475, 0.0002994651060943421], "gradient_norm": 0.5228012204170227, "seconds": 0.296303383999998, "nonpadding_tokens_per_second": 246375.8564431397, "ranks": [{"compute_seconds": 0.28742628099996637, "nonpadding_tokens": 9343, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2892601119999654, "nonpadding_tokens": 17544, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29362069699993754, "nonpadding_tokens": 11972, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.291997570000035, "nonpadding_tokens": 9305, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29236860599996817, "nonpadding_tokens": 5593, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28818993599998066, "nonpadding_tokens": 1566, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28527890499992736, "nonpadding_tokens": 10797, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28552025700003014, "nonpadding_tokens": 6882, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040820000003805035, "prefetch": {"data_wait_seconds": 0.015058081001328105, "consumed_batches": 1606, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2156, "tokens": 161229887, "loss_tokens": 72078, "loss": 2.65807939836011, "lr": 0.00029946456006130245, "optimizer_lrs": [0.01996430400408683, 0.00029946456006130245], "gradient_norm": 0.4467396140098572, "seconds": 0.34183127800008606, "nonpadding_tokens_per_second": 210858.41068055175, "ranks": [{"compute_seconds": 0.33512417899999036, "nonpadding_tokens": 8982, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3317327539999724, "nonpadding_tokens": 4665, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3345370790000288, "nonpadding_tokens": 15711, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32976022599996213, "nonpadding_tokens": 5181, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33900110699994457, "nonpadding_tokens": 12244, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3326573010000402, "nonpadding_tokens": 11489, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32976588300005005, "nonpadding_tokens": 10121, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3278022350000356, "nonpadding_tokens": 3685, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003577979999818126, "prefetch": {"data_wait_seconds": 0.015066311001376675, "consumed_batches": 1607, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2157, "tokens": 161298406, "loss_tokens": 68519, "loss": 2.4356834418190574, "lr": 0.0002994640407320167, "optimizer_lrs": [0.01996426938213445, 0.0002994640407320167], "gradient_norm": 1.4008162021636963, "seconds": 0.2993823599999814, "nonpadding_tokens_per_second": 228867.859816471, "ranks": [{"compute_seconds": 0.29410046699990744, "nonpadding_tokens": 6356, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29368197599990253, "nonpadding_tokens": 3088, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29105109199997514, "nonpadding_tokens": 15792, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.291120228000068, "nonpadding_tokens": 16791, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28983057599998574, "nonpadding_tokens": 2828, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2946885129999828, "nonpadding_tokens": 8089, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2967240260001063, "nonpadding_tokens": 12006, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2901956840000821, "nonpadding_tokens": 3569, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038700300001437427, "prefetch": {"data_wait_seconds": 0.015075186001311636, "consumed_batches": 1608, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2158, "tokens": 161403197, "loss_tokens": 104791, "loss": 2.2460006703819984, "lr": 0.00029946324599873216, "optimizer_lrs": [0.01996421639991548, 0.00029946324599873216], "gradient_norm": 0.6926030516624451, "seconds": 0.31490326199991614, "nonpadding_tokens_per_second": 332772.0371471665, "ranks": [{"compute_seconds": 0.2300324330000194, "nonpadding_tokens": 15599, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25923740700000053, "nonpadding_tokens": 10518, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2564593000000741, "nonpadding_tokens": 11099, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25698278400000163, "nonpadding_tokens": 7092, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26383315600003243, "nonpadding_tokens": 11078, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25801025200007643, "nonpadding_tokens": 21461, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2552922730000091, "nonpadding_tokens": 9426, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2639972519999674, "nonpadding_tokens": 18518, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.03536739399999078, "prefetch": {"data_wait_seconds": 0.015083379001296748, "consumed_batches": 1609, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2159, "tokens": 161464706, "loss_tokens": 61509, "loss": 2.7588325488952834, "lr": 0.0002994627792417684, "optimizer_lrs": [0.019964185282784562, 0.0002994627792417684], "gradient_norm": 0.4601175785064697, "seconds": 0.37933756500001437, "nonpadding_tokens_per_second": 162148.45476745145, "ranks": [{"compute_seconds": 0.35411979900004553, "nonpadding_tokens": 2206, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36291827000002286, "nonpadding_tokens": 5151, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35989659900008064, "nonpadding_tokens": 1876, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35960772999999335, "nonpadding_tokens": 12208, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3674470299999939, "nonpadding_tokens": 17594, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36113412099996367, "nonpadding_tokens": 10419, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3652921319999223, "nonpadding_tokens": 5830, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3589469190000045, "nonpadding_tokens": 6225, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004910239999844634, "prefetch": {"data_wait_seconds": 0.015093075001303191, "consumed_batches": 1610, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2160, "tokens": 161539714, "loss_tokens": 75008, "loss": 2.270264921741681, "lr": 0.0002994622097745664, "optimizer_lrs": [0.019964147318304426, 0.0002994622097745664], "gradient_norm": 0.42732396721839905, "seconds": 0.3016913599999498, "nonpadding_tokens_per_second": 248624.95233543476, "ranks": [{"compute_seconds": 0.2578366919999553, "nonpadding_tokens": 5344, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2599440760000107, "nonpadding_tokens": 3178, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26290075199995044, "nonpadding_tokens": 11684, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2632688279999229, "nonpadding_tokens": 6482, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2566482240000596, "nonpadding_tokens": 9926, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2610661369999434, "nonpadding_tokens": 5982, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2565257260000635, "nonpadding_tokens": 16948, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.266173933999994, "nonpadding_tokens": 15464, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.020503898000015397, "prefetch": {"data_wait_seconds": 0.015102374001301655, "consumed_batches": 1611, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2161, "tokens": 161600980, "loss_tokens": 61266, "loss": 2.2136906073515488, "lr": 0.0002994617444144332, "optimizer_lrs": [0.019964116294295547, 0.0002994617444144332], "gradient_norm": 0.3865678906440735, "seconds": 0.298740227000053, "nonpadding_tokens_per_second": 205081.1858022359, "ranks": [{"compute_seconds": 0.29294602999993913, "nonpadding_tokens": 2798, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29200911299994914, "nonpadding_tokens": 9598, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2945382810000865, "nonpadding_tokens": 4213, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.290586511000015, "nonpadding_tokens": 12777, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2885126469999477, "nonpadding_tokens": 9950, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2928363820000186, "nonpadding_tokens": 12083, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.288837689999923, "nonpadding_tokens": 4788, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29622301400002016, "nonpadding_tokens": 5059, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033962899999551155, "prefetch": {"data_wait_seconds": 0.015110791001305302, "consumed_batches": 1612, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2162, "tokens": 161662539, "loss_tokens": 61559, "loss": 2.585554356389805, "lr": 0.0002994612766264541, "optimizer_lrs": [0.019964085108430275, 0.0002994612766264541], "gradient_norm": 0.39650213718414307, "seconds": 0.2718817660000923, "nonpadding_tokens_per_second": 226418.27330185543, "ranks": [{"compute_seconds": 0.2233665019999762, "nonpadding_tokens": 3480, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2256584770000245, "nonpadding_tokens": 7284, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2227035949999845, "nonpadding_tokens": 3185, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22346710700003314, "nonpadding_tokens": 4217, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2288156979999485, "nonpadding_tokens": 12998, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22451844199997595, "nonpadding_tokens": 3287, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22842259199990167, "nonpadding_tokens": 17946, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22187088999999105, "nonpadding_tokens": 9162, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.02583955300008256, "prefetch": {"data_wait_seconds": 0.01512170600138063, "consumed_batches": 1613, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2163, "tokens": 161776194, "loss_tokens": 113655, "loss": 2.104761531608816, "lr": 0.0002994604124272065, "optimizer_lrs": [0.019964027495147103, 0.0002994604124272065], "gradient_norm": 0.3908941149711609, "seconds": 0.33617564999997285, "nonpadding_tokens_per_second": 338082.19006941514, "ranks": [{"compute_seconds": 0.3325003209999977, "nonpadding_tokens": 10070, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3296366739999712, "nonpadding_tokens": 20483, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32612629400000515, "nonpadding_tokens": 10486, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3274990269999307, "nonpadding_tokens": 11997, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33291949399995246, "nonpadding_tokens": 2482, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33043823899993185, "nonpadding_tokens": 12120, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3255648949999568, "nonpadding_tokens": 16851, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33363167799996063, "nonpadding_tokens": 29166, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000473000000056345, "prefetch": {"data_wait_seconds": 0.015133000001313235, "consumed_batches": 1614, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2164, "tokens": 161856825, "loss_tokens": 80631, "loss": 2.7427241848668626, "lr": 0.0002994597989136276, "optimizer_lrs": [0.019963986594241843, 0.0002994597989136276], "gradient_norm": 0.35974398255348206, "seconds": 0.29138340100007554, "nonpadding_tokens_per_second": 276717.89032340626, "ranks": [{"compute_seconds": 0.2834311330000219, "nonpadding_tokens": 6267, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2842755240000088, "nonpadding_tokens": 11760, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27970564000008835, "nonpadding_tokens": 14656, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28725967499997296, "nonpadding_tokens": 20051, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27961184200000844, "nonpadding_tokens": 11197, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2849278549999781, "nonpadding_tokens": 10053, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2788142480000033, "nonpadding_tokens": 3211, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2886483980000776, "nonpadding_tokens": 3436, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035042000001794804, "prefetch": {"data_wait_seconds": 0.015141455001256077, "consumed_batches": 1615, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2165, "tokens": 161945558, "loss_tokens": 88733, "loss": 2.2862273618608633, "lr": 0.000299459123350627, "optimizer_lrs": [0.019963941556708466, 0.000299459123350627], "gradient_norm": 0.31370872259140015, "seconds": 0.3193660130000353, "nonpadding_tokens_per_second": 277841.0863650359, "ranks": [{"compute_seconds": 0.286463542999968, "nonpadding_tokens": 15634, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2990572050000537, "nonpadding_tokens": 12088, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29604744599998867, "nonpadding_tokens": 7177, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29657155400002466, "nonpadding_tokens": 14098, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29499238000005334, "nonpadding_tokens": 5548, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2982956149999154, "nonpadding_tokens": 15138, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3026624039999888, "nonpadding_tokens": 9139, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29567397699997855, "nonpadding_tokens": 9911, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01117704099999628, "prefetch": {"data_wait_seconds": 0.015152251001225636, "consumed_batches": 1616, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2166, "tokens": 162003091, "loss_tokens": 57533, "loss": 2.5525152412528462, "lr": 0.00029945868510167906, "optimizer_lrs": [0.01996391234011194, 0.00029945868510167906], "gradient_norm": 0.4340752065181732, "seconds": 0.31193876899999395, "nonpadding_tokens_per_second": 184436.83734611748, "ranks": [{"compute_seconds": 0.30412283800001205, "nonpadding_tokens": 9955, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3054704570000695, "nonpadding_tokens": 6724, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30218438899999, "nonpadding_tokens": 11171, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3083034960000077, "nonpadding_tokens": 2631, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30246500800001286, "nonpadding_tokens": 6124, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3040608769999835, "nonpadding_tokens": 14805, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3088322840000046, "nonpadding_tokens": 656, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30944372199996906, "nonpadding_tokens": 5467, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041993700006059953, "prefetch": {"data_wait_seconds": 0.015161712001216188, "consumed_batches": 1617, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2167, "tokens": 162073297, "loss_tokens": 70206, "loss": 2.412174529242515, "lr": 0.00029945815007812837, "optimizer_lrs": [0.019963876671875228, 0.00029945815007812837], "gradient_norm": 0.3426542282104492, "seconds": 0.28971936699997514, "nonpadding_tokens_per_second": 242324.15225457132, "ranks": [{"compute_seconds": 0.28716802499991445, "nonpadding_tokens": 8475, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2832933659999526, "nonpadding_tokens": 12715, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2814363779999667, "nonpadding_tokens": 5279, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28144033000000945, "nonpadding_tokens": 10023, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28035363300000427, "nonpadding_tokens": 5997, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2840891929999998, "nonpadding_tokens": 17461, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28007770099998197, "nonpadding_tokens": 6397, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2805975160000571, "nonpadding_tokens": 3859, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035243299998910516, "prefetch": {"data_wait_seconds": 0.015169642001254147, "consumed_batches": 1618, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2168, "tokens": 162142672, "loss_tokens": 69375, "loss": 2.083273873873874, "lr": 0.00029945762112835864, "optimizer_lrs": [0.019963841408557243, 0.00029945762112835864], "gradient_norm": 0.34014350175857544, "seconds": 0.3550216510000155, "nonpadding_tokens_per_second": 195410.61736540956, "ranks": [{"compute_seconds": 0.3520838709999907, "nonpadding_tokens": 12253, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34942729699992015, "nonpadding_tokens": 3744, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34661475699999755, "nonpadding_tokens": 7456, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.347579775999975, "nonpadding_tokens": 11114, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3459119719999535, "nonpadding_tokens": 1809, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3486208860000488, "nonpadding_tokens": 9945, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34639420399992105, "nonpadding_tokens": 12631, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3467769620000354, "nonpadding_tokens": 10423, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043557799995141977, "prefetch": {"data_wait_seconds": 0.015179556001228889, "consumed_batches": 1619, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2169, "tokens": 162203802, "loss_tokens": 61130, "loss": 2.421910273188287, "lr": 0.00029945715482916006, "optimizer_lrs": [0.019963810321944005, 0.00029945715482916006], "gradient_norm": 0.5283991694450378, "seconds": 0.2849527070000022, "nonpadding_tokens_per_second": 214526.82672707344, "ranks": [{"compute_seconds": 0.2813451490000034, "nonpadding_tokens": 14048, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2784233749999885, "nonpadding_tokens": 10445, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28123025100001087, "nonpadding_tokens": 5094, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27519216199993934, "nonpadding_tokens": 4895, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2823349479999706, "nonpadding_tokens": 4403, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2767520520000062, "nonpadding_tokens": 13818, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.273881981000045, "nonpadding_tokens": 5122, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2747941349999792, "nonpadding_tokens": 3305, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003725860000258763, "prefetch": {"data_wait_seconds": 0.015187684001219282, "consumed_batches": 1620, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2170, "tokens": 162283821, "loss_tokens": 80019, "loss": 2.5706070589484997, "lr": 0.00029945654414262584, "optimizer_lrs": [0.01996376960950839, 0.00029945654414262584], "gradient_norm": 0.370367169380188, "seconds": 0.2992454300000418, "nonpadding_tokens_per_second": 267402.57988230203, "ranks": [{"compute_seconds": 0.29109263700001975, "nonpadding_tokens": 17082, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2937461640000265, "nonpadding_tokens": 10439, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2906738530000439, "nonpadding_tokens": 10909, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2914590410000528, "nonpadding_tokens": 11751, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29656998699999804, "nonpadding_tokens": 8367, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2950930569999173, "nonpadding_tokens": 9518, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2959592380000231, "nonpadding_tokens": 6531, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2899575400000458, "nonpadding_tokens": 5422, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043106800001169177, "prefetch": {"data_wait_seconds": 0.015197287001186055, "consumed_batches": 1621, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2171, "tokens": 162367349, "loss_tokens": 83528, "loss": 2.1065161158533665, "lr": 0.0002994559063107304, "optimizer_lrs": [0.01996372708738203, 0.0002994559063107304], "gradient_norm": 0.5541269183158875, "seconds": 0.3346142300000565, "nonpadding_tokens_per_second": 249624.76939485176, "ranks": [{"compute_seconds": 0.329635630000098, "nonpadding_tokens": 8163, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3297569620000331, "nonpadding_tokens": 5510, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32712645100002646, "nonpadding_tokens": 13952, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3320390349999798, "nonpadding_tokens": 3765, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32653522700002213, "nonpadding_tokens": 4880, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3299559780000436, "nonpadding_tokens": 9996, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32561265899994396, "nonpadding_tokens": 8591, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3266291680000677, "nonpadding_tokens": 28671, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034054799994009954, "prefetch": {"data_wait_seconds": 0.015206662001176596, "consumed_batches": 1622, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2172, "tokens": 162441979, "loss_tokens": 74630, "loss": 2.365939049309929, "lr": 0.00029945533610946254, "optimizer_lrs": [0.01996368907396417, 0.00029945533610946254], "gradient_norm": 0.3925502896308899, "seconds": 0.3610141479999811, "nonpadding_tokens_per_second": 206723.20022206972, "ranks": [{"compute_seconds": 0.35640628099997684, "nonpadding_tokens": 9489, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3546631780000098, "nonpadding_tokens": 11616, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3584021810000877, "nonpadding_tokens": 10559, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35724177800000234, "nonpadding_tokens": 11813, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3508878430000095, "nonpadding_tokens": 5835, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35372230099994795, "nonpadding_tokens": 9709, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35087225099994157, "nonpadding_tokens": 7276, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3519518779999089, "nonpadding_tokens": 8333, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003694340000492957, "prefetch": {"data_wait_seconds": 0.015215362001143262, "consumed_batches": 1623, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2173, "tokens": 162484366, "loss_tokens": 42387, "loss": 2.2382172010286174, "lr": 0.00029945501212414106, "optimizer_lrs": [0.01996366747494274, 0.00029945501212414106], "gradient_norm": 0.41329169273376465, "seconds": 0.28985639500001525, "nonpadding_tokens_per_second": 146234.48276860605, "ranks": [{"compute_seconds": 0.28526905199998964, "nonpadding_tokens": 10583, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2836580399999775, "nonpadding_tokens": 3309, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2862179770000921, "nonpadding_tokens": 7416, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28314968500001214, "nonpadding_tokens": 2286, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2871972599999708, "nonpadding_tokens": 7068, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28550012400000924, "nonpadding_tokens": 2742, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28095306900002015, "nonpadding_tokens": 4438, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28133532900005775, "nonpadding_tokens": 4545, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004490149999583082, "prefetch": {"data_wait_seconds": 0.015225402001149746, "consumed_batches": 1624, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2174, "tokens": 162530813, "loss_tokens": 46447, "loss": 2.2853134352057185, "lr": 0.0002994546569957933, "optimizer_lrs": [0.019963643799719558, 0.0002994546569957933], "gradient_norm": 0.41450944542884827, "seconds": 0.3053116689999342, "nonpadding_tokens_per_second": 152129.7897068258, "ranks": [{"compute_seconds": 0.2995816709999417, "nonpadding_tokens": 10560, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2996705699999893, "nonpadding_tokens": 2570, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29606745000000956, "nonpadding_tokens": 9218, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29633745299997827, "nonpadding_tokens": 2792, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3017616109999608, "nonpadding_tokens": 3541, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29920854800002417, "nonpadding_tokens": 4676, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29532872199990834, "nonpadding_tokens": 6797, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30281508799998846, "nonpadding_tokens": 6293, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041095699998550117, "prefetch": {"data_wait_seconds": 0.015234620001095323, "consumed_batches": 1625, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2175, "tokens": 162602408, "loss_tokens": 71595, "loss": 2.3286223723723722, "lr": 0.00029945410936266627, "optimizer_lrs": [0.01996360729084442, 0.00029945410936266627], "gradient_norm": 0.3606697916984558, "seconds": 0.28292499999997744, "nonpadding_tokens_per_second": 253052.92922154532, "ranks": [{"compute_seconds": 0.27685141299991756, "nonpadding_tokens": 6669, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.276755496000078, "nonpadding_tokens": 6498, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27893016299992723, "nonpadding_tokens": 2489, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2749529760000087, "nonpadding_tokens": 21117, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2802571070000113, "nonpadding_tokens": 9462, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2757477039999685, "nonpadding_tokens": 12299, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2730980640000098, "nonpadding_tokens": 6171, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27421284500007914, "nonpadding_tokens": 6890, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041537700008120737, "prefetch": {"data_wait_seconds": 0.015243568001054086, "consumed_batches": 1626, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2176, "tokens": 162679152, "loss_tokens": 76744, "loss": 2.7548102294641925, "lr": 0.0002994535220400559, "optimizer_lrs": [0.01996356813600373, 0.0002994535220400559], "gradient_norm": 0.6530714631080627, "seconds": 0.2967582459999676, "nonpadding_tokens_per_second": 258607.80967147375, "ranks": [{"compute_seconds": 0.2913144750000356, "nonpadding_tokens": 2640, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2910596250000026, "nonpadding_tokens": 5977, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29271276899999066, "nonpadding_tokens": 9751, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29318717800003924, "nonpadding_tokens": 11107, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29005184699997244, "nonpadding_tokens": 8626, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28940337100004854, "nonpadding_tokens": 7597, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28706795999994483, "nonpadding_tokens": 23641, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29427331500005494, "nonpadding_tokens": 7405, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000385998000069776, "prefetch": {"data_wait_seconds": 0.015251167001068211, "consumed_batches": 1627, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2177, "tokens": 162723004, "loss_tokens": 43852, "loss": 2.5081347843313875, "lr": 0.00029945318629877175, "optimizer_lrs": [0.01996354575325145, 0.00029945318629877175], "gradient_norm": 0.4478832483291626, "seconds": 0.3093542609999531, "nonpadding_tokens_per_second": 141753.34084054088, "ranks": [{"compute_seconds": 0.30678996999995434, "nonpadding_tokens": 6045, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30221420999998827, "nonpadding_tokens": 4907, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2990755860000718, "nonpadding_tokens": 8183, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3062621860000263, "nonpadding_tokens": 2659, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2976165149999588, "nonpadding_tokens": 5197, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3006901920000473, "nonpadding_tokens": 5772, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2976417789999459, "nonpadding_tokens": 5045, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29839246799997454, "nonpadding_tokens": 6044, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00048015400000167574, "prefetch": {"data_wait_seconds": 0.015261660001058885, "consumed_batches": 1628, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2178, "tokens": 162783770, "loss_tokens": 60766, "loss": 2.399657343333443, "lr": 0.00029945272088982447, "optimizer_lrs": [0.0199635147259883, 0.00029945272088982447], "gradient_norm": 0.43491336703300476, "seconds": 0.27468283600001087, "nonpadding_tokens_per_second": 221222.41376595368, "ranks": [{"compute_seconds": 0.27126308499998686, "nonpadding_tokens": 4539, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2686299230000486, "nonpadding_tokens": 2008, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27128634899997905, "nonpadding_tokens": 5516, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27216552899994895, "nonpadding_tokens": 9210, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26478672899997946, "nonpadding_tokens": 6756, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26771604800001114, "nonpadding_tokens": 3782, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26441303400008564, "nonpadding_tokens": 6093, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26490261799995096, "nonpadding_tokens": 22862, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037518299996008864, "prefetch": {"data_wait_seconds": 0.015271003001046779, "consumed_batches": 1629, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2179, "tokens": 162835218, "loss_tokens": 51448, "loss": 2.454665997706422, "lr": 0.00029945232669331646, "optimizer_lrs": [0.0199634884462211, 0.00029945232669331646], "gradient_norm": 1.1660664081573486, "seconds": 0.33099888200001715, "nonpadding_tokens_per_second": 155432.54916491633, "ranks": [{"compute_seconds": 0.32742082499999015, "nonpadding_tokens": 11365, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3241796050000403, "nonpadding_tokens": 3076, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32265443999995114, "nonpadding_tokens": 13635, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3230431660000477, "nonpadding_tokens": 4328, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3218023850000691, "nonpadding_tokens": 4374, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3251404670000966, "nonpadding_tokens": 10243, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32842330000005404, "nonpadding_tokens": 2214, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32187927799998306, "nonpadding_tokens": 2213, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034568299997772556, "prefetch": {"data_wait_seconds": 0.015280110001029357, "consumed_batches": 1630, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2180, "tokens": 162909343, "loss_tokens": 74125, "loss": 2.46920826306914, "lr": 0.0002994517584957595, "optimizer_lrs": [0.019963450566383968, 0.0002994517584957595], "gradient_norm": 0.48661887645721436, "seconds": 0.32130790600001546, "nonpadding_tokens_per_second": 230697.7158538901, "ranks": [{"compute_seconds": 0.3175515840000571, "nonpadding_tokens": 16993, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3161394950000158, "nonpadding_tokens": 14854, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31174166400001013, "nonpadding_tokens": 7824, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.312915170999986, "nonpadding_tokens": 5053, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3184081160000005, "nonpadding_tokens": 7584, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31530833599993, "nonpadding_tokens": 5590, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.318300153999985, "nonpadding_tokens": 10823, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31187874199997623, "nonpadding_tokens": 5404, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004282710000325096, "prefetch": {"data_wait_seconds": 0.015290198001096655, "consumed_batches": 1631, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2181, "tokens": 162999064, "loss_tokens": 89721, "loss": 2.4490091157588525, "lr": 0.00029945107035538493, "optimizer_lrs": [0.019963404690358996, 0.00029945107035538493], "gradient_norm": 0.5299184918403625, "seconds": 0.3267425229998935, "nonpadding_tokens_per_second": 274592.3584608828, "ranks": [{"compute_seconds": 0.31673431700005494, "nonpadding_tokens": 9346, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3199091840000392, "nonpadding_tokens": 4378, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3155860729999631, "nonpadding_tokens": 16879, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32281926300004216, "nonpadding_tokens": 16443, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3234846989998914, "nonpadding_tokens": 16554, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3205007959999193, "nonpadding_tokens": 5843, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31478022900000724, "nonpadding_tokens": 5947, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3242066390000673, "nonpadding_tokens": 14331, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000425744999915878, "prefetch": {"data_wait_seconds": 0.015301692001187917, "consumed_batches": 1632, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2182, "tokens": 163043794, "loss_tokens": 44730, "loss": 2.0441974765258215, "lr": 0.00029945072712521045, "optimizer_lrs": [0.019963381808347364, 0.00029945072712521045], "gradient_norm": 0.3965570032596588, "seconds": 0.2284506879999526, "nonpadding_tokens_per_second": 195797.1779012952, "ranks": [{"compute_seconds": 0.18956849300002432, "nonpadding_tokens": 4034, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20735220499989282, "nonpadding_tokens": 2815, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20318535699993845, "nonpadding_tokens": 2748, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20402688600006513, "nonpadding_tokens": 17391, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20250040200005515, "nonpadding_tokens": 4664, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20522034199996142, "nonpadding_tokens": 2850, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2026185429999714, "nonpadding_tokens": 4671, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20312643599993407, "nonpadding_tokens": 5557, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.008882706999997936, "prefetch": {"data_wait_seconds": 0.015309842001173024, "consumed_batches": 1633, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2183, "tokens": 163112643, "loss_tokens": 68849, "loss": 2.530567346657177, "lr": 0.0002994501986116408, "optimizer_lrs": [0.01996334657410939, 0.0002994501986116408], "gradient_norm": 0.4029200077056885, "seconds": 0.36074988799998664, "nonpadding_tokens_per_second": 190849.67810163964, "ranks": [{"compute_seconds": 0.35451703299997916, "nonpadding_tokens": 12458, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3545454659999905, "nonpadding_tokens": 8137, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3579987450000317, "nonpadding_tokens": 13629, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35373766500003967, "nonpadding_tokens": 5585, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.357930830999976, "nonpadding_tokens": 8438, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35515756400002374, "nonpadding_tokens": 6901, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.357640908999997, "nonpadding_tokens": 11543, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35327883600007226, "nonpadding_tokens": 2158, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005525130000023637, "prefetch": {"data_wait_seconds": 0.015318637001158208, "consumed_batches": 1634, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2184, "tokens": 163159206, "loss_tokens": 46563, "loss": 2.4043849193565707, "lr": 0.0002994498410309108, "optimizer_lrs": [0.019963322735394053, 0.0002994498410309108], "gradient_norm": 1.3057401180267334, "seconds": 0.2413651229999232, "nonpadding_tokens_per_second": 192915.19595403524, "ranks": [{"compute_seconds": 0.23812212500001806, "nonpadding_tokens": 4916, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23567441000000144, "nonpadding_tokens": 13192, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23121743300009712, "nonpadding_tokens": 5665, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2317052399999966, "nonpadding_tokens": 4077, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2387920060000397, "nonpadding_tokens": 3719, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23325880799995957, "nonpadding_tokens": 3013, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23071233800010305, "nonpadding_tokens": 9519, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23125874400000157, "nonpadding_tokens": 2462, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037612500000250293, "prefetch": {"data_wait_seconds": 0.015327397001172471, "consumed_batches": 1635, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2185, "tokens": 163225285, "loss_tokens": 66079, "loss": 2.1112255311823724, "lr": 0.0002994493333778576, "optimizer_lrs": [0.019963288891857178, 0.0002994493333778576], "gradient_norm": 0.3856849670410156, "seconds": 0.2901937409999391, "nonpadding_tokens_per_second": 227706.49626110945, "ranks": [{"compute_seconds": 0.28271608399995785, "nonpadding_tokens": 11938, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2845772880000368, "nonpadding_tokens": 8029, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28081575499993505, "nonpadding_tokens": 25529, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2866127149999329, "nonpadding_tokens": 3386, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2874889610000082, "nonpadding_tokens": 2732, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2833537570000999, "nonpadding_tokens": 7820, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28697091999993063, "nonpadding_tokens": 2267, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28012132300000303, "nonpadding_tokens": 4378, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046478600006594206, "prefetch": {"data_wait_seconds": 0.015336775001131286, "consumed_batches": 1636, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2186, "tokens": 163270185, "loss_tokens": 44900, "loss": 2.3579906737193763, "lr": 0.00029944898829952773, "optimizer_lrs": [0.019963265886635183, 0.00029944898829952773], "gradient_norm": 0.4125169515609741, "seconds": 0.3053794669999661, "nonpadding_tokens_per_second": 147030.1865449421, "ranks": [{"compute_seconds": 0.30214817800003857, "nonpadding_tokens": 6118, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2998531319999529, "nonpadding_tokens": 5267, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3020107209999878, "nonpadding_tokens": 10815, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29716142899997067, "nonpadding_tokens": 2343, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2960378640000272, "nonpadding_tokens": 5724, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2983599220000315, "nonpadding_tokens": 4428, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30270238200000676, "nonpadding_tokens": 2284, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2965247199999794, "nonpadding_tokens": 7921, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000461964999999509, "prefetch": {"data_wait_seconds": 0.015363475001166194, "consumed_batches": 1637, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2187, "tokens": 163326741, "loss_tokens": 56556, "loss": 2.3613199748921425, "lr": 0.0002994485534856649, "optimizer_lrs": [0.01996323689904433, 0.0002994485534856649], "gradient_norm": 0.36749181151390076, "seconds": 0.28258231299992076, "nonpadding_tokens_per_second": 200139.91463087735, "ranks": [{"compute_seconds": 0.2793505739999773, "nonpadding_tokens": 10472, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27604739900004915, "nonpadding_tokens": 9186, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2784911659999807, "nonpadding_tokens": 5901, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27908539399993515, "nonpadding_tokens": 3743, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2716903340000272, "nonpadding_tokens": 3429, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27481722300001366, "nonpadding_tokens": 8696, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2790675449999753, "nonpadding_tokens": 8983, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27247248700007276, "nonpadding_tokens": 6146, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0008256689999370792, "prefetch": {"data_wait_seconds": 0.015373342001183232, "consumed_batches": 1638, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2188, "tokens": 163366766, "loss_tokens": 40025, "loss": 2.3519952373516553, "lr": 0.00029944824566202665, "optimizer_lrs": [0.019963216377468444, 0.00029944824566202665], "gradient_norm": 0.41476815938949585, "seconds": 0.32545786899993345, "nonpadding_tokens_per_second": 122980.5876963085, "ranks": [{"compute_seconds": 0.3228211120000424, "nonpadding_tokens": 4555, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3216396169999598, "nonpadding_tokens": 4773, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3187193059999345, "nonpadding_tokens": 5527, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3222027629999502, "nonpadding_tokens": 2800, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3223510210000313, "nonpadding_tokens": 5508, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31952054400005636, "nonpadding_tokens": 8506, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3220505580000008, "nonpadding_tokens": 5076, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3178386730000966, "nonpadding_tokens": 3280, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040000400008466386, "prefetch": {"data_wait_seconds": 0.015384746001203098, "consumed_batches": 1639, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2189, "tokens": 163414954, "loss_tokens": 48188, "loss": 2.6540770329231345, "lr": 0.0002994478749447965, "optimizer_lrs": [0.019963191662986435, 0.0002994478749447965], "gradient_norm": 0.44178223609924316, "seconds": 0.3266638370000692, "nonpadding_tokens_per_second": 147515.56353019204, "ranks": [{"compute_seconds": 0.3231049340000709, "nonpadding_tokens": 4510, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3200120249999827, "nonpadding_tokens": 4481, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3166707120000183, "nonpadding_tokens": 5122, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3171677190000537, "nonpadding_tokens": 7855, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32399045600004683, "nonpadding_tokens": 9707, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3185591110000132, "nonpadding_tokens": 6171, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31548263600006976, "nonpadding_tokens": 7333, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3159773810000388, "nonpadding_tokens": 3009, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004202869999971881, "prefetch": {"data_wait_seconds": 0.015394992001191099, "consumed_batches": 1640, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2190, "tokens": 163486047, "loss_tokens": 71093, "loss": 2.255294121784142, "lr": 0.00029944732778923715, "optimizer_lrs": [0.019963155185949145, 0.00029944732778923715], "gradient_norm": 0.5339862108230591, "seconds": 0.33925141400004577, "nonpadding_tokens_per_second": 209558.44859055005, "ranks": [{"compute_seconds": 0.33504907600001843, "nonpadding_tokens": 9403, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3350157659999695, "nonpadding_tokens": 18780, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33195096999997986, "nonpadding_tokens": 7024, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3330135400000245, "nonpadding_tokens": 4714, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33145258100000774, "nonpadding_tokens": 7036, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33391932899996846, "nonpadding_tokens": 8825, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33665340200002447, "nonpadding_tokens": 9653, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3366356540000197, "nonpadding_tokens": 5658, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004690329999448295, "prefetch": {"data_wait_seconds": 0.015405629001136845, "consumed_batches": 1641, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2191, "tokens": 163519664, "loss_tokens": 33617, "loss": 2.2842483899515127, "lr": 0.0002994470689673667, "optimizer_lrs": [0.01996313793115778, 0.0002994470689673667], "gradient_norm": 0.47451040148735046, "seconds": 0.33056478399998923, "nonpadding_tokens_per_second": 101695.64825756241, "ranks": [{"compute_seconds": 0.32703322600002593, "nonpadding_tokens": 4704, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32562457299991365, "nonpadding_tokens": 4272, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3223459019999382, "nonpadding_tokens": 4204, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32254275599996163, "nonpadding_tokens": 4054, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3276219630000696, "nonpadding_tokens": 4545, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3239602730000115, "nonpadding_tokens": 913, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3269262219999973, "nonpadding_tokens": 2451, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3280124909999813, "nonpadding_tokens": 8474, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003943030000073122, "prefetch": {"data_wait_seconds": 0.015415514001077213, "consumed_batches": 1642, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2192, "tokens": 163589338, "loss_tokens": 69674, "loss": 2.4640869890489996, "lr": 0.00029944653234520414, "optimizer_lrs": [0.019963102156346942, 0.00029944653234520414], "gradient_norm": 0.39645811915397644, "seconds": 0.30012654300003305, "nonpadding_tokens_per_second": 232148.74400493238, "ranks": [{"compute_seconds": 0.29665590800004793, "nonpadding_tokens": 3562, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2930921770000623, "nonpadding_tokens": 4866, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2960960719999548, "nonpadding_tokens": 2616, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2907478509999919, "nonpadding_tokens": 13784, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28918894100002035, "nonpadding_tokens": 17071, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29266354100002445, "nonpadding_tokens": 9134, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2896937149999985, "nonpadding_tokens": 12254, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2975206429999844, "nonpadding_tokens": 6387, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004290699999955905, "prefetch": {"data_wait_seconds": 0.015441414001045928, "consumed_batches": 1643, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2193, "tokens": 163668646, "loss_tokens": 79308, "loss": 2.0764560715816813, "lr": 0.00029944592120684905, "optimizer_lrs": [0.019963061413789937, 0.00029944592120684905], "gradient_norm": 0.3487696647644043, "seconds": 0.2995868259999952, "nonpadding_tokens_per_second": 264724.5910606272, "ranks": [{"compute_seconds": 0.29661451800006944, "nonpadding_tokens": 3171, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2926683419999563, "nonpadding_tokens": 11102, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29523423400007687, "nonpadding_tokens": 2287, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2900001109999266, "nonpadding_tokens": 9784, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2886372630000551, "nonpadding_tokens": 22227, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2914497570000094, "nonpadding_tokens": 8097, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29562166400000933, "nonpadding_tokens": 2146, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2966180759999588, "nonpadding_tokens": 20494, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034890500000983593, "prefetch": {"data_wait_seconds": 0.015451072000999488, "consumed_batches": 1644, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2194, "tokens": 163725976, "loss_tokens": 57330, "loss": 2.1059353468951683, "lr": 0.0002994454792188802, "optimizer_lrs": [0.01996303194792535, 0.0002994454792188802], "gradient_norm": 0.3475196659564972, "seconds": 0.33757461000004696, "nonpadding_tokens_per_second": 169829.12310849453, "ranks": [{"compute_seconds": 0.3349623149999843, "nonpadding_tokens": 13014, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3318370800000139, "nonpadding_tokens": 4692, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3290433459999349, "nonpadding_tokens": 5908, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33025806299997384, "nonpadding_tokens": 4972, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3287371510000412, "nonpadding_tokens": 9224, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33115842400002293, "nonpadding_tokens": 1852, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32813885900009154, "nonpadding_tokens": 4373, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3287042059999976, "nonpadding_tokens": 13295, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004432090000818789, "prefetch": {"data_wait_seconds": 0.015460609001095804, "consumed_batches": 1645, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2195, "tokens": 163764678, "loss_tokens": 38702, "loss": 2.0867417220040307, "lr": 0.00029944518074480813, "optimizer_lrs": [0.019963012049653877, 0.00029944518074480813], "gradient_norm": 0.4800766706466675, "seconds": 0.2909018999999944, "nonpadding_tokens_per_second": 133041.41361744542, "ranks": [{"compute_seconds": 0.2872413959998994, "nonpadding_tokens": 2562, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28479061699999875, "nonpadding_tokens": 4325, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2828516280000031, "nonpadding_tokens": 5029, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28814434900004926, "nonpadding_tokens": 4593, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28834991399992305, "nonpadding_tokens": 9617, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2853762199999892, "nonpadding_tokens": 3085, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28171401199995216, "nonpadding_tokens": 4830, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2825324279999677, "nonpadding_tokens": 4661, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004252759999872069, "prefetch": {"data_wait_seconds": 0.015470694001194829, "consumed_batches": 1646, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2196, "tokens": 163844872, "loss_tokens": 80194, "loss": 2.473319777664164, "lr": 0.00029944456202480436, "optimizer_lrs": [0.019962970801653625, 0.00029944456202480436], "gradient_norm": 0.4214443564414978, "seconds": 0.3475429239999812, "nonpadding_tokens_per_second": 230745.59849189833, "ranks": [{"compute_seconds": 0.34348141600003146, "nonpadding_tokens": 6285, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34247312700006205, "nonpadding_tokens": 6482, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34438962800004447, "nonpadding_tokens": 11135, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3391489220000494, "nonpadding_tokens": 19310, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.337525305999975, "nonpadding_tokens": 13525, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34235251799998423, "nonpadding_tokens": 11442, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3450182220000215, "nonpadding_tokens": 9371, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33788637499992547, "nonpadding_tokens": 2644, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004755079999085865, "prefetch": {"data_wait_seconds": 0.015481079001190778, "consumed_batches": 1647, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2197, "tokens": 163926259, "loss_tokens": 81387, "loss": 2.745336325211643, "lr": 0.00029944393374865596, "optimizer_lrs": [0.019962928916577066, 0.00029944393374865596], "gradient_norm": 0.4450867176055908, "seconds": 0.3311626629999864, "nonpadding_tokens_per_second": 245761.40094634803, "ranks": [{"compute_seconds": 0.32461643900001036, "nonpadding_tokens": 16267, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.323141856999996, "nonpadding_tokens": 3125, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32049828999993224, "nonpadding_tokens": 6689, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3210547769999721, "nonpadding_tokens": 3224, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.326157706999993, "nonpadding_tokens": 17029, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32333432600000833, "nonpadding_tokens": 6128, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32599967499993454, "nonpadding_tokens": 17438, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32866276999993715, "nonpadding_tokens": 11487, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042008900004475436, "prefetch": {"data_wait_seconds": 0.01549041000119189, "consumed_batches": 1648, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2198, "tokens": 163996233, "loss_tokens": 69974, "loss": 2.490000080387001, "lr": 0.00029944339329311464, "optimizer_lrs": [0.019962892886207645, 0.00029944339329311464], "gradient_norm": 0.4938367009162903, "seconds": 0.5783433089999335, "nonpadding_tokens_per_second": 120990.42024191214, "ranks": [{"compute_seconds": 0.5719028019999541, "nonpadding_tokens": 5216, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5714485079998894, "nonpadding_tokens": 5490, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.5702538399999639, "nonpadding_tokens": 3101, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.5746861410000292, "nonpadding_tokens": 10053, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.5687884530000247, "nonpadding_tokens": 11544, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5723280649999651, "nonpadding_tokens": 11604, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.5690186880000283, "nonpadding_tokens": 19008, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.575730973000077, "nonpadding_tokens": 3958, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004200650000711903, "prefetch": {"data_wait_seconds": 0.015499365001190313, "consumed_batches": 1649, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2199, "tokens": 164090027, "loss_tokens": 93794, "loss": 2.4620078309913214, "lr": 0.0002994426684490216, "optimizer_lrs": [0.019962844563268106, 0.0002994426684490216], "gradient_norm": 0.37470412254333496, "seconds": 0.6316892349999534, "nonpadding_tokens_per_second": 148481.23856346373, "ranks": [{"compute_seconds": 0.61702947699996, "nonpadding_tokens": 11478, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.6175111499999275, "nonpadding_tokens": 18047, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.6157881730000554, "nonpadding_tokens": 14412, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.6212259919999497, "nonpadding_tokens": 5150, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.6213220990000536, "nonpadding_tokens": 15252, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.6178261229999862, "nonpadding_tokens": 6724, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.6148044440000149, "nonpadding_tokens": 16988, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.6216094860000112, "nonpadding_tokens": 5743, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040479599999798666, "prefetch": {"data_wait_seconds": 0.01550971200113338, "consumed_batches": 1650, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2200, "tokens": 164177251, "loss_tokens": 87224, "loss": 2.3675450564064935, "lr": 0.00029944199395579035, "optimizer_lrs": [0.019962799597052692, 0.00029944199395579035], "gradient_norm": 0.4102221429347992, "seconds": 0.3353288729999804, "nonpadding_tokens_per_second": 260114.79184497515, "ranks": [{"compute_seconds": 0.3093920089999074, "nonpadding_tokens": 7556, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3285600950000571, "nonpadding_tokens": 5943, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3254537440000149, "nonpadding_tokens": 15567, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32574041600003056, "nonpadding_tokens": 21565, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3325337109999964, "nonpadding_tokens": 9408, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3296548849999681, "nonpadding_tokens": 3030, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3327338230000123, "nonpadding_tokens": 10703, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3245892489999278, "nonpadding_tokens": 13452, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003444330000093032, "prefetch": {"data_wait_seconds": 0.015517604001118457, "consumed_batches": 1651, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2201, "tokens": 164244232, "loss_tokens": 66981, "loss": 2.183911762290799, "lr": 0.00029944147572305666, "optimizer_lrs": [0.019962765048203778, 0.00029944147572305666], "gradient_norm": 0.4278241991996765, "seconds": 0.2749801509999088, "nonpadding_tokens_per_second": 243584.85423925094, "ranks": [{"compute_seconds": 0.2719068569999763, "nonpadding_tokens": 13955, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26893520699991313, "nonpadding_tokens": 7810, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2712912150000193, "nonpadding_tokens": 9367, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.265996255999994, "nonpadding_tokens": 10705, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2643801409999469, "nonpadding_tokens": 5540, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26703143100007765, "nonpadding_tokens": 11302, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27214813399996274, "nonpadding_tokens": 3099, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26477276100001745, "nonpadding_tokens": 5203, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00048667199996543786, "prefetch": {"data_wait_seconds": 0.015528194001220186, "consumed_batches": 1652, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2202, "tokens": 164311716, "loss_tokens": 67484, "loss": 2.469813678057021, "lr": 0.00029944095335585915, "optimizer_lrs": [0.019962730223723944, 0.00029944095335585915], "gradient_norm": 0.3638036847114563, "seconds": 0.2272334059999821, "nonpadding_tokens_per_second": 296980.9817488073, "ranks": [{"compute_seconds": 0.21967331999996986, "nonpadding_tokens": 4804, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2202180029998999, "nonpadding_tokens": 18705, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21916532700004154, "nonpadding_tokens": 14135, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2195548619999954, "nonpadding_tokens": 10615, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2236023399999567, "nonpadding_tokens": 4868, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22004449200005638, "nonpadding_tokens": 4406, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2177974590000531, "nonpadding_tokens": 6354, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22467145200005234, "nonpadding_tokens": 3597, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043803699998079537, "prefetch": {"data_wait_seconds": 0.015539598001240051, "consumed_batches": 1653, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2203, "tokens": 164385448, "loss_tokens": 73732, "loss": 2.3819911130852276, "lr": 0.00029944038234680536, "optimizer_lrs": [0.01996269215645369, 0.00029944038234680536], "gradient_norm": 0.4168441891670227, "seconds": 0.34195475599995007, "nonpadding_tokens_per_second": 215619.16805160846, "ranks": [{"compute_seconds": 0.3386372879999726, "nonpadding_tokens": 15932, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3351536960000203, "nonpadding_tokens": 19605, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3316097719999789, "nonpadding_tokens": 5780, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33210187899999255, "nonpadding_tokens": 2634, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33936037600005875, "nonpadding_tokens": 4124, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33379920100003346, "nonpadding_tokens": 7949, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3308295699999917, "nonpadding_tokens": 13890, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3381717810000282, "nonpadding_tokens": 3818, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033424000002924004, "prefetch": {"data_wait_seconds": 0.015547383001219828, "consumed_batches": 1654, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2204, "tokens": 164564849, "loss_tokens": 179401, "loss": 2.3069851199268676, "lr": 0.0002994389917814241, "optimizer_lrs": [0.019962599452094942, 0.0002994389917814241], "gradient_norm": 0.4462660253047943, "seconds": 0.3457733889999872, "nonpadding_tokens_per_second": 518839.8115854041, "ranks": [{"compute_seconds": 0.3378540720000274, "nonpadding_tokens": 28242, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3401206839999986, "nonpadding_tokens": 18007, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3427513240000053, "nonpadding_tokens": 12783, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33783825399996203, "nonpadding_tokens": 17169, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3364585159999933, "nonpadding_tokens": 19900, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34096618000000944, "nonpadding_tokens": 27765, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34332585700008167, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33614935400009927, "nonpadding_tokens": 22767, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039336200006800937, "prefetch": {"data_wait_seconds": 0.015557119001144315, "consumed_batches": 1655, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2205, "tokens": 164655375, "loss_tokens": 90526, "loss": 2.3322697001413957, "lr": 0.0002994382894464859, "optimizer_lrs": [0.019962552629765727, 0.0002994382894464859], "gradient_norm": 0.454731822013855, "seconds": 0.3403357590000269, "nonpadding_tokens_per_second": 265990.26874514483, "ranks": [{"compute_seconds": 0.3362080359999027, "nonpadding_tokens": 25185, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.333683548999943, "nonpadding_tokens": 13876, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3329121970000415, "nonpadding_tokens": 19016, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33234701799995037, "nonpadding_tokens": 12618, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33006291399999554, "nonpadding_tokens": 6723, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33487953199994536, "nonpadding_tokens": 6431, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.330282976000035, "nonpadding_tokens": 3311, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3377001679999694, "nonpadding_tokens": 3366, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037701299993386783, "prefetch": {"data_wait_seconds": 0.015564460001158409, "consumed_batches": 1656, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2206, "tokens": 164709108, "loss_tokens": 53733, "loss": 2.784219834645376, "lr": 0.0002994378723581972, "optimizer_lrs": [0.019962524823879814, 0.0002994378723581972], "gradient_norm": 0.9500753283500671, "seconds": 0.31935436499998104, "nonpadding_tokens_per_second": 168255.09806325394, "ranks": [{"compute_seconds": 0.30635614699997404, "nonpadding_tokens": 2978, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3076164039999867, "nonpadding_tokens": 9567, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30440080199991826, "nonpadding_tokens": 3195, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3106842949999873, "nonpadding_tokens": 7536, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31103660999997373, "nonpadding_tokens": 18891, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3063306769998917, "nonpadding_tokens": 1937, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31077444499999274, "nonpadding_tokens": 5844, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31625920599992696, "nonpadding_tokens": 3785, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046777600005043496, "prefetch": {"data_wait_seconds": 0.015575651001157098, "consumed_batches": 1657, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2207, "tokens": 164783237, "loss_tokens": 74129, "loss": 2.541885387297819, "lr": 0.0002994372966977704, "optimizer_lrs": [0.01996248644651803, 0.0002994372966977704], "gradient_norm": 0.8340467214584351, "seconds": 0.30259262699996725, "nonpadding_tokens_per_second": 244979.53150724992, "ranks": [{"compute_seconds": 0.29986579299998084, "nonpadding_tokens": 9943, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29602329599993027, "nonpadding_tokens": 9485, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2942734879999307, "nonpadding_tokens": 17343, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2985248570000749, "nonpadding_tokens": 4012, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2933337140000276, "nonpadding_tokens": 10521, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29588263599998754, "nonpadding_tokens": 2447, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2935700040000029, "nonpadding_tokens": 10127, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.293879832000016, "nonpadding_tokens": 10251, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044804499998463143, "prefetch": {"data_wait_seconds": 0.015587318001237804, "consumed_batches": 1658, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2208, "tokens": 164854899, "loss_tokens": 71662, "loss": 2.7173312302894144, "lr": 0.000299436739915741, "optimizer_lrs": [0.01996244932771607, 0.000299436739915741], "gradient_norm": 1.6848605871200562, "seconds": 0.27740868900002624, "nonpadding_tokens_per_second": 258326.4434085308, "ranks": [{"compute_seconds": 0.27477952100002767, "nonpadding_tokens": 6122, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2703836059999958, "nonpadding_tokens": 12589, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2738044230000014, "nonpadding_tokens": 9266, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27467708100004984, "nonpadding_tokens": 11017, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26837617699993643, "nonpadding_tokens": 6999, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27176332500005174, "nonpadding_tokens": 8115, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2665447829999721, "nonpadding_tokens": 12058, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26736363700001675, "nonpadding_tokens": 5496, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000433402999988175, "prefetch": {"data_wait_seconds": 0.015598120001257598, "consumed_batches": 1659, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2209, "tokens": 164905794, "loss_tokens": 50895, "loss": 2.3816274621770313, "lr": 0.0002994363443172559, "optimizer_lrs": [0.01996242295448373, 0.0002994363443172559], "gradient_norm": 0.4713118374347687, "seconds": 0.3993969370000059, "nonpadding_tokens_per_second": 127429.62022264895, "ranks": [{"compute_seconds": 0.3951903070000071, "nonpadding_tokens": 18525, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.39291956099998515, "nonpadding_tokens": 5545, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3898131020000619, "nonpadding_tokens": 9272, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3955283850000342, "nonpadding_tokens": 4173, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3887726359999988, "nonpadding_tokens": 2092, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3912832270000308, "nonpadding_tokens": 4613, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.38825193600007424, "nonpadding_tokens": 3474, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3966401909999604, "nonpadding_tokens": 3201, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000491979999992509, "prefetch": {"data_wait_seconds": 0.015607851001234962, "consumed_batches": 1660, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2210, "tokens": 164970229, "loss_tokens": 64435, "loss": 2.347282629781951, "lr": 0.0002994358432757834, "optimizer_lrs": [0.019962389551718897, 0.0002994358432757834], "gradient_norm": 0.40861445665359497, "seconds": 0.2848054010000851, "nonpadding_tokens_per_second": 226242.19826498567, "ranks": [{"compute_seconds": 0.2759788580000304, "nonpadding_tokens": 6828, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27900872199995774, "nonpadding_tokens": 7747, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27977753300001496, "nonpadding_tokens": 6692, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2756800500000054, "nonpadding_tokens": 1632, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2735254030000078, "nonpadding_tokens": 6351, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27740101000006234, "nonpadding_tokens": 6759, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27419283699998687, "nonpadding_tokens": 10451, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28185142300003463, "nonpadding_tokens": 17975, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005417610000222339, "prefetch": {"data_wait_seconds": 0.015615960001198914, "consumed_batches": 1661, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2211, "tokens": 165020604, "loss_tokens": 50375, "loss": 2.296211228287841, "lr": 0.00029943545140904986, "optimizer_lrs": [0.019962363427269994, 0.00029943545140904986], "gradient_norm": 0.4191431403160095, "seconds": 0.20924853899998652, "nonpadding_tokens_per_second": 240742.4216233273, "ranks": [{"compute_seconds": 0.20497546700005387, "nonpadding_tokens": 6157, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20296171099994353, "nonpadding_tokens": 5931, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20031833100006224, "nonpadding_tokens": 4599, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20572100300000784, "nonpadding_tokens": 1712, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.19968588899996575, "nonpadding_tokens": 7814, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20203903999993145, "nonpadding_tokens": 7356, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19924747400000342, "nonpadding_tokens": 4866, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20665899799996623, "nonpadding_tokens": 11940, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004172210000206178, "prefetch": {"data_wait_seconds": 0.015624627001173508, "consumed_batches": 1662, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2212, "tokens": 165080086, "loss_tokens": 59482, "loss": 2.4797057828418683, "lr": 0.00029943498852423546, "optimizer_lrs": [0.019962332568282365, 0.00029943498852423546], "gradient_norm": 0.6672716736793518, "seconds": 0.2853605029999926, "nonpadding_tokens_per_second": 208445.10496255168, "ranks": [{"compute_seconds": 0.27942169400000694, "nonpadding_tokens": 7220, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28097490100003597, "nonpadding_tokens": 2415, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2825184270000136, "nonpadding_tokens": 9732, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2794499590000896, "nonpadding_tokens": 6617, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2777719650000563, "nonpadding_tokens": 2022, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2802236100000073, "nonpadding_tokens": 5049, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28267265399995267, "nonpadding_tokens": 7597, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27949889599995004, "nonpadding_tokens": 18830, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041811500000221713, "prefetch": {"data_wait_seconds": 0.015635894001164274, "consumed_batches": 1663, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2213, "tokens": 165121917, "loss_tokens": 41831, "loss": 2.2187735321890463, "lr": 0.0002994346628849164, "optimizer_lrs": [0.019962310858994428, 0.0002994346628849164], "gradient_norm": 0.4345771074295044, "seconds": 0.2607393350000393, "nonpadding_tokens_per_second": 160432.25698950904, "ranks": [{"compute_seconds": 0.2566375330000028, "nonpadding_tokens": 9583, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25518842399992536, "nonpadding_tokens": 2824, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25552756100000806, "nonpadding_tokens": 1032, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2565647999999783, "nonpadding_tokens": 1697, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2582093209999812, "nonpadding_tokens": 14365, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25525262200005727, "nonpadding_tokens": 4180, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2542096299999912, "nonpadding_tokens": 1564, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2550323870000284, "nonpadding_tokens": 6586, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038861500001985405, "prefetch": {"data_wait_seconds": 0.01564499100113892, "consumed_batches": 1664, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2214, "tokens": 165231392, "loss_tokens": 109475, "loss": 3.9571057889929206, "lr": 0.00029943381021824855, "optimizer_lrs": [0.019962254014549905, 0.00029943381021824855], "gradient_norm": 1.7124812602996826, "seconds": 0.29889995199994246, "nonpadding_tokens_per_second": 366259.67741882097, "ranks": [{"compute_seconds": 0.2959634969998888, "nonpadding_tokens": 4641, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2933269720000453, "nonpadding_tokens": 17537, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2954740849999098, "nonpadding_tokens": 15418, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29615567300004386, "nonpadding_tokens": 25007, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2887035549999837, "nonpadding_tokens": 13638, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29178461999993033, "nonpadding_tokens": 22326, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2885752089999869, "nonpadding_tokens": 3304, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2889899130000231, "nonpadding_tokens": 7604, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004373699999860037, "prefetch": {"data_wait_seconds": 0.015654643001155932, "consumed_batches": 1665, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2215, "tokens": 165270459, "loss_tokens": 39067, "loss": 2.3599617324084265, "lr": 0.00029943350578232557, "optimizer_lrs": [0.019962233718821706, 0.00029943350578232557], "gradient_norm": 0.4424474835395813, "seconds": 0.27076923099991745, "nonpadding_tokens_per_second": 144281.53396798586, "ranks": [{"compute_seconds": 0.26601928699994914, "nonpadding_tokens": 2099, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26456164100000024, "nonpadding_tokens": 3629, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2653011009999773, "nonpadding_tokens": 3918, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26297679499998594, "nonpadding_tokens": 5734, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2659315680000418, "nonpadding_tokens": 3413, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26380705600001875, "nonpadding_tokens": 2838, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2603829140000471, "nonpadding_tokens": 4230, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2680931119999741, "nonpadding_tokens": 13206, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044217200002094614, "prefetch": {"data_wait_seconds": 0.01566453800121792, "consumed_batches": 1666, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2216, "tokens": 165350108, "loss_tokens": 79649, "loss": 2.579658289809037, "lr": 0.00029943288485166246, "optimizer_lrs": [0.019962192323444164, 0.00029943288485166246], "gradient_norm": 0.42452266812324524, "seconds": 0.2772688630000175, "nonpadding_tokens_per_second": 287262.69202465395, "ranks": [{"compute_seconds": 0.27446527099994, "nonpadding_tokens": 7839, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27167596300000696, "nonpadding_tokens": 10275, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2691794300000083, "nonpadding_tokens": 3753, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26969755400000395, "nonpadding_tokens": 4946, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.268024913999966, "nonpadding_tokens": 3288, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2701623440000276, "nonpadding_tokens": 20507, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2747488690000637, "nonpadding_tokens": 19145, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.268827205999969, "nonpadding_tokens": 9896, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004437040000766501, "prefetch": {"data_wait_seconds": 0.015676020001208713, "consumed_batches": 1667, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2217, "tokens": 165393086, "loss_tokens": 42978, "loss": 2.6553200183814973, "lr": 0.00029943254966117994, "optimizer_lrs": [0.019962169977411997, 0.00029943254966117994], "gradient_norm": 0.4446941614151001, "seconds": 0.2575848149999729, "nonpadding_tokens_per_second": 166849.89757647214, "ranks": [{"compute_seconds": 0.25067303100001936, "nonpadding_tokens": 7816, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2513215050000781, "nonpadding_tokens": 3987, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24762008099992272, "nonpadding_tokens": 5817, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2545067990000689, "nonpadding_tokens": 3236, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24751290799997605, "nonpadding_tokens": 3452, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25288877399998455, "nonpadding_tokens": 9429, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2549374399999351, "nonpadding_tokens": 3956, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24788746799993078, "nonpadding_tokens": 5285, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003419220000751011, "prefetch": {"data_wait_seconds": 0.015684359001241432, "consumed_batches": 1668, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2218, "tokens": 165440425, "loss_tokens": 47339, "loss": 2.5581393314708802, "lr": 0.00029943218034437705, "optimizer_lrs": [0.019962145356291806, 0.00029943218034437705], "gradient_norm": 0.5659038424491882, "seconds": 0.2774948249999625, "nonpadding_tokens_per_second": 170594.17234179558, "ranks": [{"compute_seconds": 0.23296056799995313, "nonpadding_tokens": 5785, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24693485299997064, "nonpadding_tokens": 5529, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24368520899997748, "nonpadding_tokens": 9415, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24526348200004122, "nonpadding_tokens": 4041, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24827910399994835, "nonpadding_tokens": 982, "peak_allocated_bytes": 14980221440, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24604395500000464, "nonpadding_tokens": 7210, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2480500450000136, "nonpadding_tokens": 12154, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2489403099999663, "nonpadding_tokens": 2223, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.014596048000043993, "prefetch": {"data_wait_seconds": 0.01569414400121616, "consumed_batches": 1669, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2219, "tokens": 165507491, "loss_tokens": 67066, "loss": 2.6119368420660245, "lr": 0.00029943165692147947, "optimizer_lrs": [0.019962110461431967, 0.00029943165692147947], "gradient_norm": 0.45683568716049194, "seconds": 0.4931364859999121, "nonpadding_tokens_per_second": 135998.8601614279, "ranks": [{"compute_seconds": 0.48506749199998467, "nonpadding_tokens": 13183, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4855264940000552, "nonpadding_tokens": 13483, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4829972039999575, "nonpadding_tokens": 2817, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.48426402799998414, "nonpadding_tokens": 9860, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.48279896799999733, "nonpadding_tokens": 13129, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.48662558299997727, "nonpadding_tokens": 6839, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.49018990999991274, "nonpadding_tokens": 1363, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4826472710000189, "nonpadding_tokens": 6392, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004743700000062745, "prefetch": {"data_wait_seconds": 0.0157045460012597, "consumed_batches": 1670, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2220, "tokens": 165558486, "loss_tokens": 50995, "loss": 2.037453120403961, "lr": 0.0002994312587651664, "optimizer_lrs": [0.019962083917677762, 0.0002994312587651664], "gradient_norm": 0.3931836187839508, "seconds": 0.29203852099999494, "nonpadding_tokens_per_second": 174617.37521948648, "ranks": [{"compute_seconds": 0.2872673819999818, "nonpadding_tokens": 14297, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28525202200000876, "nonpadding_tokens": 10940, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2879863260000093, "nonpadding_tokens": 5239, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2836407530000997, "nonpadding_tokens": 3486, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28936832099998355, "nonpadding_tokens": 1701, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2874446710000029, "nonpadding_tokens": 1825, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28193630999999186, "nonpadding_tokens": 5191, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28208838299997296, "nonpadding_tokens": 8316, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004738680000855311, "prefetch": {"data_wait_seconds": 0.01571492400121599, "consumed_batches": 1671, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2221, "tokens": 165612115, "loss_tokens": 53629, "loss": 2.5664428877100076, "lr": 0.0002994308398931478, "optimizer_lrs": [0.019962055992876523, 0.0002994308398931478], "gradient_norm": 0.41544008255004883, "seconds": 0.285239512999965, "nonpadding_tokens_per_second": 188013.92358290343, "ranks": [{"compute_seconds": 0.2654221970000208, "nonpadding_tokens": 16154, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2786403830000381, "nonpadding_tokens": 7630, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2744576190000316, "nonpadding_tokens": 8239, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2751166489999832, "nonpadding_tokens": 3274, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27357004099997084, "nonpadding_tokens": 9482, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27877277400000366, "nonpadding_tokens": 4051, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27332644299997355, "nonpadding_tokens": 2673, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28252930400003606, "nonpadding_tokens": 2126, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004162829999359019, "prefetch": {"data_wait_seconds": 0.015724170001206517, "consumed_batches": 1672, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2222, "tokens": 165668694, "loss_tokens": 56579, "loss": 2.7061973081885506, "lr": 0.00029943039781321173, "optimizer_lrs": [0.019962026520880784, 0.00029943039781321173], "gradient_norm": 1.2081263065338135, "seconds": 0.2749521549999372, "nonpadding_tokens_per_second": 205777.61974628977, "ranks": [{"compute_seconds": 0.26500250599997344, "nonpadding_tokens": 3937, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26738165999995545, "nonpadding_tokens": 3376, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2639339209999889, "nonpadding_tokens": 2934, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26510309300010704, "nonpadding_tokens": 6544, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2706259560000035, "nonpadding_tokens": 6741, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26775070600001527, "nonpadding_tokens": 9403, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2634105309999768, "nonpadding_tokens": 19752, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2720655229999238, "nonpadding_tokens": 3892, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004189549999864539, "prefetch": {"data_wait_seconds": 0.015734349001107262, "consumed_batches": 1673, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2223, "tokens": 165765349, "loss_tokens": 96655, "loss": 2.5141644250168125, "lr": 0.000299429642203331, "optimizer_lrs": [0.019961976146888737, 0.000299429642203331], "gradient_norm": 0.4896828532218933, "seconds": 0.37796512600004917, "nonpadding_tokens_per_second": 255724.65116791602, "ranks": [{"compute_seconds": 0.3744038620000083, "nonpadding_tokens": 23130, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37280683299991324, "nonpadding_tokens": 12411, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36958037600004445, "nonpadding_tokens": 12168, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.37042857500000537, "nonpadding_tokens": 25514, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3744670800000449, "nonpadding_tokens": 4715, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3711130949999415, "nonpadding_tokens": 3234, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36864531900005204, "nonpadding_tokens": 2597, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37504639799999495, "nonpadding_tokens": 12886, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0006674830000292786, "prefetch": {"data_wait_seconds": 0.015741269001026126, "consumed_batches": 1674, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2224, "tokens": 165845099, "loss_tokens": 79750, "loss": 2.2851671238244515, "lr": 0.0002994290183736233, "optimizer_lrs": [0.019961934558241554, 0.0002994290183736233], "gradient_norm": 0.38697993755340576, "seconds": 0.28522774700002174, "nonpadding_tokens_per_second": 279601.12870783894, "ranks": [{"compute_seconds": 0.28225509899993995, "nonpadding_tokens": 20285, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2785679720000189, "nonpadding_tokens": 9561, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2811116180000681, "nonpadding_tokens": 3093, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27648305700006404, "nonpadding_tokens": 9783, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2745678930000395, "nonpadding_tokens": 12563, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2776194389999773, "nonpadding_tokens": 2203, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27435186200000317, "nonpadding_tokens": 12296, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2823412110000163, "nonpadding_tokens": 9966, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036168299993732944, "prefetch": {"data_wait_seconds": 0.015749730001019202, "consumed_batches": 1675, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2225, "tokens": 165887981, "loss_tokens": 42882, "loss": 2.650542841401987, "lr": 0.0002994286827964009, "optimizer_lrs": [0.019961912186426727, 0.0002994286827964009], "gradient_norm": 0.42441508173942566, "seconds": 0.27670648000002984, "nonpadding_tokens_per_second": 154972.87956536247, "ranks": [{"compute_seconds": 0.2739113059999454, "nonpadding_tokens": 2611, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27003896700000496, "nonpadding_tokens": 5781, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2670925569999554, "nonpadding_tokens": 2155, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27336487699994905, "nonpadding_tokens": 7177, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2662317169999824, "nonpadding_tokens": 3811, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2682824610000125, "nonpadding_tokens": 7311, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26585829000009653, "nonpadding_tokens": 6019, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2668214679999892, "nonpadding_tokens": 8017, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047654000002239627, "prefetch": {"data_wait_seconds": 0.015760156001036307, "consumed_batches": 1676, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2226, "tokens": 165961076, "loss_tokens": 73095, "loss": 2.5278840891989875, "lr": 0.0002994281105576783, "optimizer_lrs": [0.019961874037178556, 0.0002994281105576783], "gradient_norm": 1.0821727514266968, "seconds": 0.34320612200008327, "nonpadding_tokens_per_second": 212976.97014851694, "ranks": [{"compute_seconds": 0.33431821800002126, "nonpadding_tokens": 19754, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3368231489999971, "nonpadding_tokens": 4208, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33397871800002576, "nonpadding_tokens": 5421, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33453164200000174, "nonpadding_tokens": 6754, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33278562800001055, "nonpadding_tokens": 12224, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.335876417999998, "nonpadding_tokens": 13668, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33953257800010306, "nonpadding_tokens": 8459, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3406822019999254, "nonpadding_tokens": 2607, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004430169999523059, "prefetch": {"data_wait_seconds": 0.01577067200101112, "consumed_batches": 1677, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2227, "tokens": 166033847, "loss_tokens": 72771, "loss": 2.4785377674485716, "lr": 0.00029942754057152255, "optimizer_lrs": [0.019961836038101505, 0.00029942754057152255], "gradient_norm": 0.4434310495853424, "seconds": 0.3827445299999681, "nonpadding_tokens_per_second": 190129.43176485386, "ranks": [{"compute_seconds": 0.3747182340000563, "nonpadding_tokens": 17840, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37699188700003106, "nonpadding_tokens": 9068, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3790159879999919, "nonpadding_tokens": 21717, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.37402981900004306, "nonpadding_tokens": 3984, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3797002889999703, "nonpadding_tokens": 9710, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37709611299999324, "nonpadding_tokens": 1784, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37270108400002755, "nonpadding_tokens": 6502, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.38022239600002194, "nonpadding_tokens": 2166, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038911399997232365, "prefetch": {"data_wait_seconds": 0.01577922800090903, "consumed_batches": 1678, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2228, "tokens": 166087276, "loss_tokens": 53429, "loss": 2.1753318364090664, "lr": 0.0002994271219031634, "optimizer_lrs": [0.01996180812687756, 0.0002994271219031634], "gradient_norm": 0.49589812755584717, "seconds": 0.28016367699990496, "nonpadding_tokens_per_second": 190706.37768656257, "ranks": [{"compute_seconds": 0.27655357300000105, "nonpadding_tokens": 1628, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27291314300009617, "nonpadding_tokens": 6936, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27632615800007443, "nonpadding_tokens": 2390, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2707271330000367, "nonpadding_tokens": 7424, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26975228300000254, "nonpadding_tokens": 9751, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2742535330000919, "nonpadding_tokens": 13560, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2691567939999686, "nonpadding_tokens": 8606, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2776218380000728, "nonpadding_tokens": 3134, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033793700004025595, "prefetch": {"data_wait_seconds": 0.015788251000870446, "consumed_batches": 1679, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2229, "tokens": 166134493, "loss_tokens": 47217, "loss": 2.1297127822076796, "lr": 0.00029942675178477304, "optimizer_lrs": [0.019961783452318204, 0.00029942675178477304], "gradient_norm": 0.5300369262695312, "seconds": 0.251735730000064, "nonpadding_tokens_per_second": 187565.74603052175, "ranks": [{"compute_seconds": 0.2435562730000811, "nonpadding_tokens": 12137, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24578924499996901, "nonpadding_tokens": 3083, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24839400999996997, "nonpadding_tokens": 3942, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24313589799999136, "nonpadding_tokens": 2542, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24928798500002358, "nonpadding_tokens": 3032, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24669325600007141, "nonpadding_tokens": 5451, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.241359258999978, "nonpadding_tokens": 12276, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24189589899992825, "nonpadding_tokens": 4754, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004248060000691112, "prefetch": {"data_wait_seconds": 0.015799216000871752, "consumed_batches": 1680, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2230, "tokens": 166186453, "loss_tokens": 51960, "loss": 2.2935855104888376, "lr": 0.00029942634434973947, "optimizer_lrs": [0.019961756289982632, 0.00029942634434973947], "gradient_norm": 0.45899954438209534, "seconds": 0.2874882260000504, "nonpadding_tokens_per_second": 180737.83654705528, "ranks": [{"compute_seconds": 0.28508416700003636, "nonpadding_tokens": 6520, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2810275009999259, "nonpadding_tokens": 11546, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2788673549999885, "nonpadding_tokens": 7298, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2790487290000101, "nonpadding_tokens": 2702, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27765698100006375, "nonpadding_tokens": 7455, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2809248949999983, "nonpadding_tokens": 6846, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28425820600000407, "nonpadding_tokens": 5415, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2850221529999999, "nonpadding_tokens": 4178, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003471479999461735, "prefetch": {"data_wait_seconds": 0.015808482000878143, "consumed_batches": 1681, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2231, "tokens": 166247914, "loss_tokens": 61461, "loss": 2.488488126616879, "lr": 0.0002994258622278549, "optimizer_lrs": [0.019961724148523662, 0.0002994258622278549], "gradient_norm": 0.45065221190452576, "seconds": 0.3085150299999668, "nonpadding_tokens_per_second": 199215.5779250256, "ranks": [{"compute_seconds": 0.30210844999999154, "nonpadding_tokens": 8339, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3040881140000238, "nonpadding_tokens": 4284, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3051788279999528, "nonpadding_tokens": 11305, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3012436839999282, "nonpadding_tokens": 2378, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30588564399999996, "nonpadding_tokens": 17570, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30290707099993597, "nonpadding_tokens": 5695, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29899559400007547, "nonpadding_tokens": 7741, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3006024679999655, "nonpadding_tokens": 4149, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045023300003776967, "prefetch": {"data_wait_seconds": 0.015819023000929633, "consumed_batches": 1682, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2232, "tokens": 166339137, "loss_tokens": 91223, "loss": 2.5213472411014766, "lr": 0.00029942514626972965, "optimizer_lrs": [0.019961676417981977, 0.00029942514626972965], "gradient_norm": 0.38271579146385193, "seconds": 0.28263760300001195, "nonpadding_tokens_per_second": 322756.0629998555, "ranks": [{"compute_seconds": 0.20053081899993686, "nonpadding_tokens": 5129, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21086737100006303, "nonpadding_tokens": 18169, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20774473500000568, "nonpadding_tokens": 4486, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21422285400001329, "nonpadding_tokens": 14134, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21426393100000496, "nonpadding_tokens": 3909, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20962527999995473, "nonpadding_tokens": 4518, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21411181000007673, "nonpadding_tokens": 19856, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20727246399997057, "nonpadding_tokens": 21022, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.04293899000003876, "prefetch": {"data_wait_seconds": 0.015829927001007604, "consumed_batches": 1683, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2233, "tokens": 166373588, "loss_tokens": 34451, "loss": 2.4014769164900875, "lr": 0.00029942487576735914, "optimizer_lrs": [0.01996165838449061, 0.00029942487576735914], "gradient_norm": 0.44804781675338745, "seconds": 0.3672796619999872, "nonpadding_tokens_per_second": 93800.45661227276, "ranks": [{"compute_seconds": 0.363301409000087, "nonpadding_tokens": 4512, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3613934299999073, "nonpadding_tokens": 3601, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36338847299998633, "nonpadding_tokens": 2889, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3595624880000514, "nonpadding_tokens": 3408, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35827698399998553, "nonpadding_tokens": 4065, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36130400299998655, "nonpadding_tokens": 7461, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35755541699995774, "nonpadding_tokens": 3148, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3646515169999702, "nonpadding_tokens": 5367, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044885900001645496, "prefetch": {"data_wait_seconds": 0.015840644001016813, "consumed_batches": 1684, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2234, "tokens": 166434708, "loss_tokens": 61120, "loss": 2.504247024296466, "lr": 0.0002994243957090849, "optimizer_lrs": [0.01996162638060566, 0.0002994243957090849], "gradient_norm": 0.4818265736103058, "seconds": 0.3553360700000212, "nonpadding_tokens_per_second": 172006.17995239366, "ranks": [{"compute_seconds": 0.3520497299999761, "nonpadding_tokens": 11373, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34978441500004465, "nonpadding_tokens": 2485, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3488634110000248, "nonpadding_tokens": 5870, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3493941450000193, "nonpadding_tokens": 3472, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3479416929999388, "nonpadding_tokens": 22608, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3503323659999751, "nonpadding_tokens": 5542, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3525738380000121, "nonpadding_tokens": 2492, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35264632000007623, "nonpadding_tokens": 7278, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003952160000153526, "prefetch": {"data_wait_seconds": 0.015849004001054823, "consumed_batches": 1685, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2235, "tokens": 166509908, "loss_tokens": 75200, "loss": 2.3700440492021277, "lr": 0.0002994238047872707, "optimizer_lrs": [0.019961586985818045, 0.0002994238047872707], "gradient_norm": 0.40979692339897156, "seconds": 0.2833006829999931, "nonpadding_tokens_per_second": 265442.353345833, "ranks": [{"compute_seconds": 0.2785089539999035, "nonpadding_tokens": 6763, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2793978380000226, "nonpadding_tokens": 17300, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2765790760000755, "nonpadding_tokens": 12322, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27691398399997524, "nonpadding_tokens": 7685, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27561363900008473, "nonpadding_tokens": 5216, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2776223040000332, "nonpadding_tokens": 8315, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27985654500002966, "nonpadding_tokens": 5777, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2805317180000202, "nonpadding_tokens": 11822, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035591500000009546, "prefetch": {"data_wait_seconds": 0.015857374001100766, "consumed_batches": 1686, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2236, "tokens": 166567021, "loss_tokens": 57113, "loss": 2.248431152933658, "lr": 0.0002994233557910267, "optimizer_lrs": [0.019961557052735113, 0.0002994233557910267], "gradient_norm": 0.4225735366344452, "seconds": 0.2950070519999599, "nonpadding_tokens_per_second": 193598.76183572641, "ranks": [{"compute_seconds": 0.28711250900005325, "nonpadding_tokens": 2927, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2880118250000123, "nonpadding_tokens": 4530, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.284647201000098, "nonpadding_tokens": 4438, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28517825299991273, "nonpadding_tokens": 3882, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2920013819999667, "nonpadding_tokens": 3644, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2879611130000512, "nonpadding_tokens": 18035, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2903515350000134, "nonpadding_tokens": 13675, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2847671079999827, "nonpadding_tokens": 5982, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040504799994778296, "prefetch": {"data_wait_seconds": 0.015866500001109785, "consumed_batches": 1687, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2237, "tokens": 166620735, "loss_tokens": 53714, "loss": 2.4866139996090406, "lr": 0.0002994229333569359, "optimizer_lrs": [0.019961528890462394, 0.0002994229333569359], "gradient_norm": 0.4194212555885315, "seconds": 0.28650223499994354, "nonpadding_tokens_per_second": 187481.9580378163, "ranks": [{"compute_seconds": 0.2826497380000319, "nonpadding_tokens": 5988, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28038357899993116, "nonpadding_tokens": 8433, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28274738299990076, "nonpadding_tokens": 10309, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2774515479999309, "nonpadding_tokens": 2766, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28391565499998705, "nonpadding_tokens": 4159, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2790623299999879, "nonpadding_tokens": 5256, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2762799869999526, "nonpadding_tokens": 1885, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2767657759999338, "nonpadding_tokens": 14918, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042252100001860526, "prefetch": {"data_wait_seconds": 0.01587656600118237, "consumed_batches": 1688, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2238, "tokens": 166679136, "loss_tokens": 58401, "loss": 2.3025207723326657, "lr": 0.00029942247388677095, "optimizer_lrs": [0.019961498259118065, 0.00029942247388677095], "gradient_norm": 0.4109504520893097, "seconds": 0.2727619940000068, "nonpadding_tokens_per_second": 214109.74140333696, "ranks": [{"compute_seconds": 0.16702887899998586, "nonpadding_tokens": 10050, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.16719113500005278, "nonpadding_tokens": 14729, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.1646993399999701, "nonpadding_tokens": 7475, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.16447497700005442, "nonpadding_tokens": 5265, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1640090879999434, "nonpadding_tokens": 2748, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.16728315900002144, "nonpadding_tokens": 11891, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.1696873329999562, "nonpadding_tokens": 2700, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.17007553199994163, "nonpadding_tokens": 3543, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.051681754000014735, "prefetch": {"data_wait_seconds": 0.01588599900112797, "consumed_batches": 1689, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2239, "tokens": 166750696, "loss_tokens": 71560, "loss": 2.454444478409726, "lr": 0.00029942191063935474, "optimizer_lrs": [0.019961460709290318, 0.00029942191063935474], "gradient_norm": 0.43236371874809265, "seconds": 0.26747795599999336, "nonpadding_tokens_per_second": 267536.0656636758, "ranks": [{"compute_seconds": 0.26473019399998066, "nonpadding_tokens": 4919, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26328269199996157, "nonpadding_tokens": 4403, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26416117500002656, "nonpadding_tokens": 6529, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2616249359999756, "nonpadding_tokens": 6117, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2631623279999076, "nonpadding_tokens": 14493, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26206987599994136, "nonpadding_tokens": 10601, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2645843509999395, "nonpadding_tokens": 12022, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26016457600007925, "nonpadding_tokens": 12476, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044076400001813454, "prefetch": {"data_wait_seconds": 0.015897142001065845, "consumed_batches": 1690, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2240, "tokens": 166815483, "loss_tokens": 64787, "loss": 2.488623296340315, "lr": 0.000299421400465821, "optimizer_lrs": [0.019961426697721402, 0.000299421400465821], "gradient_norm": 0.3998587727546692, "seconds": 0.380028024000012, "nonpadding_tokens_per_second": 170479.53284623544, "ranks": [{"compute_seconds": 0.3760595460000786, "nonpadding_tokens": 4483, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37425742499999615, "nonpadding_tokens": 8645, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3713010389999454, "nonpadding_tokens": 5548, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3715163230000371, "nonpadding_tokens": 13662, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3766161969999757, "nonpadding_tokens": 15407, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3740178939999623, "nonpadding_tokens": 2659, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37735078799994426, "nonpadding_tokens": 9220, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3712168380000094, "nonpadding_tokens": 5163, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045275299999048, "prefetch": {"data_wait_seconds": 0.015908257001072457, "consumed_batches": 1691, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2241, "tokens": 166879008, "loss_tokens": 63525, "loss": 2.473548307752853, "lr": 0.0002994209000120505, "optimizer_lrs": [0.019961393334136702, 0.0002994209000120505], "gradient_norm": 0.38901838660240173, "seconds": 0.2508161179999888, "nonpadding_tokens_per_second": 253273.19674090017, "ranks": [{"compute_seconds": 0.24761213299996143, "nonpadding_tokens": 3008, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2464193510000996, "nonpadding_tokens": 8721, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2433217790000981, "nonpadding_tokens": 9702, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2475711989999354, "nonpadding_tokens": 10224, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2479835170000797, "nonpadding_tokens": 4542, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24552881900001466, "nonpadding_tokens": 9498, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24252006300002904, "nonpadding_tokens": 9657, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2439550290000625, "nonpadding_tokens": 8173, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040162100003726664, "prefetch": {"data_wait_seconds": 0.0159177650011344, "consumed_batches": 1692, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2242, "tokens": 166933703, "loss_tokens": 54695, "loss": 2.51684454703355, "lr": 0.0002994204689486026, "optimizer_lrs": [0.019961364596573508, 0.0002994204689486026], "gradient_norm": 0.3829284906387329, "seconds": 0.3720048960000213, "nonpadding_tokens_per_second": 147027.63481907742, "ranks": [{"compute_seconds": 0.3683699189999743, "nonpadding_tokens": 13702, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36643629600007444, "nonpadding_tokens": 7885, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3622850730000664, "nonpadding_tokens": 4801, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3684534739999208, "nonpadding_tokens": 2705, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3691868980000663, "nonpadding_tokens": 8409, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36466440300000613, "nonpadding_tokens": 8102, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36079333600002883, "nonpadding_tokens": 1875, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.36912500300002193, "nonpadding_tokens": 7216, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004544880000594276, "prefetch": {"data_wait_seconds": 0.01593004100114115, "consumed_batches": 1693, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2243, "tokens": 166981127, "loss_tokens": 47424, "loss": 2.292578981633772, "lr": 0.00029942009505998804, "optimizer_lrs": [0.01996133967066587, 0.00029942009505998804], "gradient_norm": 0.3727759122848511, "seconds": 0.24347663799994734, "nonpadding_tokens_per_second": 194778.4411250588, "ranks": [{"compute_seconds": 0.23680914100009431, "nonpadding_tokens": 1402, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23619432899999993, "nonpadding_tokens": 2494, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2393968330000007, "nonpadding_tokens": 9876, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2337819929999796, "nonpadding_tokens": 1847, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2324325700000145, "nonpadding_tokens": 4961, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23518234599998777, "nonpadding_tokens": 4191, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24077107599998726, "nonpadding_tokens": 15364, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23355389999994713, "nonpadding_tokens": 7289, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047843300001204625, "prefetch": {"data_wait_seconds": 0.015940169001126492, "consumed_batches": 1694, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2244, "tokens": 167099814, "loss_tokens": 118687, "loss": 2.256223038327702, "lr": 0.0002994191588098672, "optimizer_lrs": [0.019961277253991147, 0.0002994191588098672], "gradient_norm": 0.45916882157325745, "seconds": 0.29259874499996386, "nonpadding_tokens_per_second": 405630.5846425099, "ranks": [{"compute_seconds": 0.28825478400005977, "nonpadding_tokens": 14162, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28661467300003096, "nonpadding_tokens": 19223, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.284202233999963, "nonpadding_tokens": 13315, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28518700699999044, "nonpadding_tokens": 6434, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2898405299999922, "nonpadding_tokens": 8144, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28615929099998993, "nonpadding_tokens": 24958, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28279432700003326, "nonpadding_tokens": 10593, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2837473480000199, "nonpadding_tokens": 21858, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036652999995112623, "prefetch": {"data_wait_seconds": 0.015948878001111666, "consumed_batches": 1695, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2245, "tokens": 167151056, "loss_tokens": 51242, "loss": 2.2218553627883377, "lr": 0.0002994187543597531, "optimizer_lrs": [0.019961250290650205, 0.0002994187543597531], "gradient_norm": 0.4922742545604706, "seconds": 0.3552167840000493, "nonpadding_tokens_per_second": 144255.5709867383, "ranks": [{"compute_seconds": 0.3498191579999457, "nonpadding_tokens": 14668, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3486037090000309, "nonpadding_tokens": 2552, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35019706700006736, "nonpadding_tokens": 1737, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3457902789999707, "nonpadding_tokens": 11079, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35130352900000617, "nonpadding_tokens": 7713, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3472461869999961, "nonpadding_tokens": 3716, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35092224899995017, "nonpadding_tokens": 6221, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35262390399998367, "nonpadding_tokens": 3556, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004337979999036179, "prefetch": {"data_wait_seconds": 0.015958107001168287, "consumed_batches": 1696, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2246, "tokens": 167205144, "loss_tokens": 54088, "loss": 2.088619616412143, "lr": 0.0002994183272939485, "optimizer_lrs": [0.019961221819596565, 0.0002994183272939485], "gradient_norm": 0.4818606674671173, "seconds": 0.34192683599997054, "nonpadding_tokens_per_second": 158185.88746279237, "ranks": [{"compute_seconds": 0.331213039999966, "nonpadding_tokens": 4445, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33353059300009136, "nonpadding_tokens": 1725, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.338986324000075, "nonpadding_tokens": 4195, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.339047843000003, "nonpadding_tokens": 10078, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33869326399997135, "nonpadding_tokens": 18697, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33198598100000254, "nonpadding_tokens": 4544, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32884563800007527, "nonpadding_tokens": 1946, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3301851229999784, "nonpadding_tokens": 8458, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034466199997496005, "prefetch": {"data_wait_seconds": 0.01596580300117978, "consumed_batches": 1697, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2247, "tokens": 167264113, "loss_tokens": 58969, "loss": 2.384905363411284, "lr": 0.0002994178615106288, "optimizer_lrs": [0.019961190767375254, 0.0002994178615106288], "gradient_norm": 0.3962772488594055, "seconds": 0.31548355500001435, "nonpadding_tokens_per_second": 186916.2403726474, "ranks": [{"compute_seconds": 0.3065541810000241, "nonpadding_tokens": 4819, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3083717219999471, "nonpadding_tokens": 15256, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30515604699996857, "nonpadding_tokens": 11583, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.311172322999937, "nonpadding_tokens": 1629, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3128608700000086, "nonpadding_tokens": 7500, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30734600599998885, "nonpadding_tokens": 10210, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31148394900003495, "nonpadding_tokens": 5277, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3120179420000113, "nonpadding_tokens": 2695, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041596700009449705, "prefetch": {"data_wait_seconds": 0.015974972001188803, "consumed_batches": 1698, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2248, "tokens": 167312799, "loss_tokens": 48686, "loss": 2.492060731010968, "lr": 0.00029941747681030555, "optimizer_lrs": [0.01996116512068704, 0.00029941747681030555], "gradient_norm": 0.4361201524734497, "seconds": 0.3860525510000343, "nonpadding_tokens_per_second": 126112.36442780473, "ranks": [{"compute_seconds": 0.3833329450000065, "nonpadding_tokens": 10988, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3809029729999338, "nonpadding_tokens": 2522, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.38115581800002474, "nonpadding_tokens": 8869, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3820812349999869, "nonpadding_tokens": 6044, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3772104700000227, "nonpadding_tokens": 2209, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37984602799997447, "nonpadding_tokens": 3449, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.38192198000001554, "nonpadding_tokens": 8327, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3778001470000163, "nonpadding_tokens": 6278, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033970699996643816, "prefetch": {"data_wait_seconds": 0.015982752001221456, "consumed_batches": 1699, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2249, "tokens": 167364526, "loss_tokens": 51727, "loss": 2.4112662693080984, "lr": 0.00029941706794210475, "optimizer_lrs": [0.019961137862806986, 0.00029941706794210475], "gradient_norm": 0.40366020798683167, "seconds": 0.3668528459999152, "nonpadding_tokens_per_second": 141002.04091100866, "ranks": [{"compute_seconds": 0.3596440710000479, "nonpadding_tokens": 5030, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3612529580000228, "nonpadding_tokens": 11995, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3578738630000089, "nonpadding_tokens": 4156, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3579002600000649, "nonpadding_tokens": 4526, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35615658699998676, "nonpadding_tokens": 10006, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3594534169999406, "nonpadding_tokens": 3709, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36392685400005576, "nonpadding_tokens": 7556, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3571805320000294, "nonpadding_tokens": 4749, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005366980000189869, "prefetch": {"data_wait_seconds": 0.01599506600128109, "consumed_batches": 1700, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2250, "tokens": 167464425, "loss_tokens": 99899, "loss": 2.2223930544850297, "lr": 0.0002994162779005245, "optimizer_lrs": [0.019961085193368304, 0.0002994162779005245], "gradient_norm": 0.4277908205986023, "seconds": 0.31637093300003016, "nonpadding_tokens_per_second": 315765.41831038083, "ranks": [{"compute_seconds": 0.3115634509999836, "nonpadding_tokens": 13297, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30974037299995416, "nonpadding_tokens": 9877, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30674402399995415, "nonpadding_tokens": 13434, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3125769380000065, "nonpadding_tokens": 4277, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31322339999996984, "nonpadding_tokens": 4681, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3087816610000118, "nonpadding_tokens": 18036, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3059170090000407, "nonpadding_tokens": 15943, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.313735113000007, "nonpadding_tokens": 20354, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004431050000448522, "prefetch": {"data_wait_seconds": 0.016005635001377527, "consumed_batches": 1701, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2251, "tokens": 167560841, "loss_tokens": 96416, "loss": 2.4056821481911714, "lr": 0.0002994155148976624, "optimizer_lrs": [0.019961034326510828, 0.0002994155148976624], "gradient_norm": 0.3655968904495239, "seconds": 0.3955059700000447, "nonpadding_tokens_per_second": 243778.87393201448, "ranks": [{"compute_seconds": 0.38743863299998793, "nonpadding_tokens": 14885, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3898275469999817, "nonpadding_tokens": 7058, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.386335368999994, "nonpadding_tokens": 12756, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.38623174399992877, "nonpadding_tokens": 12136, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3851837009999599, "nonpadding_tokens": 3536, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38759506499991403, "nonpadding_tokens": 5078, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.38468267500002185, "nonpadding_tokens": 19857, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3927743529999361, "nonpadding_tokens": 21110, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005169560000695128, "prefetch": {"data_wait_seconds": 0.01601655700137883, "consumed_batches": 1702, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2252, "tokens": 167680562, "loss_tokens": 119721, "loss": 2.078644567369133, "lr": 0.00029941456677500883, "optimizer_lrs": [0.019960971118333923, 0.00029941456677500883], "gradient_norm": 0.5714778304100037, "seconds": 0.4021712449999768, "nonpadding_tokens_per_second": 297686.6235178174, "ranks": [{"compute_seconds": 0.3961223640000071, "nonpadding_tokens": 17766, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3964975210000148, "nonpadding_tokens": 8707, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3991378790000226, "nonpadding_tokens": 22317, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3959746660000292, "nonpadding_tokens": 26830, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.39392750399997567, "nonpadding_tokens": 13026, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3973243210000419, "nonpadding_tokens": 10935, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3940851580000526, "nonpadding_tokens": 14910, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.39426275499999974, "nonpadding_tokens": 5230, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003487309999172794, "prefetch": {"data_wait_seconds": 0.016024728001369226, "consumed_batches": 1703, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2253, "tokens": 167773528, "loss_tokens": 92966, "loss": 2.264434712690661, "lr": 0.00029941383000799517, "optimizer_lrs": [0.019960922000533014, 0.00029941383000799517], "gradient_norm": 0.47637343406677246, "seconds": 0.3698025080000207, "nonpadding_tokens_per_second": 251393.6438743536, "ranks": [{"compute_seconds": 0.3661736029999929, "nonpadding_tokens": 9893, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36419024999997873, "nonpadding_tokens": 2700, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3610029530000247, "nonpadding_tokens": 16943, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.366744959000016, "nonpadding_tokens": 12235, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35960982499989314, "nonpadding_tokens": 12799, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36529771899995467, "nonpadding_tokens": 12410, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35968847799995274, "nonpadding_tokens": 12176, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3673226749999685, "nonpadding_tokens": 13810, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039372000003368157, "prefetch": {"data_wait_seconds": 0.01603322800140461, "consumed_batches": 1704, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2254, "tokens": 167811147, "loss_tokens": 37619, "loss": 2.317618548805125, "lr": 0.0002994135317413933, "optimizer_lrs": [0.019960902116092888, 0.0002994135317413933], "gradient_norm": 0.4374023675918579, "seconds": 0.3081203069999674, "nonpadding_tokens_per_second": 122091.92041342468, "ranks": [{"compute_seconds": 0.305349619000026, "nonpadding_tokens": 3921, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30354743999998846, "nonpadding_tokens": 5315, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3004574509999429, "nonpadding_tokens": 4421, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30169110599990745, "nonpadding_tokens": 9226, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2995864879999317, "nonpadding_tokens": 2240, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30278192699995543, "nonpadding_tokens": 5519, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2998693109999522, "nonpadding_tokens": 3777, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30086489199993594, "nonpadding_tokens": 3200, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003520470000921705, "prefetch": {"data_wait_seconds": 0.016041993001479682, "consumed_batches": 1705, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2255, "tokens": 167900088, "loss_tokens": 88941, "loss": 2.283974418434693, "lr": 0.0002994128262613064, "optimizer_lrs": [0.019960855084087094, 0.0002994128262613064], "gradient_norm": 0.4810515344142914, "seconds": 0.3851998540000068, "nonpadding_tokens_per_second": 230895.7261442743, "ranks": [{"compute_seconds": 0.3815307390000271, "nonpadding_tokens": 9475, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37999566199994206, "nonpadding_tokens": 12993, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37674067299997205, "nonpadding_tokens": 15371, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3776420210000424, "nonpadding_tokens": 12342, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3825152119999302, "nonpadding_tokens": 9446, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.379436002000034, "nonpadding_tokens": 9905, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.382432511999923, "nonpadding_tokens": 11052, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3766656469999816, "nonpadding_tokens": 8357, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004566080000358852, "prefetch": {"data_wait_seconds": 0.016053059001436054, "consumed_batches": 1706, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2256, "tokens": 168016992, "loss_tokens": 116904, "loss": 2.5198121963320332, "lr": 0.00029941189833507226, "optimizer_lrs": [0.019960793222338152, 0.00029941189833507226], "gradient_norm": 0.4498699903488159, "seconds": 0.2906532970000626, "nonpadding_tokens_per_second": 402211.1608800186, "ranks": [{"compute_seconds": 0.28358603200001653, "nonpadding_tokens": 14496, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28449049800008197, "nonpadding_tokens": 26762, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28104995099999996, "nonpadding_tokens": 18524, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28216254699998444, "nonpadding_tokens": 17348, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28813693400002194, "nonpadding_tokens": 21548, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28299259000004895, "nonpadding_tokens": 8187, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28035843700001806, "nonpadding_tokens": 2260, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28093351400002575, "nonpadding_tokens": 7779, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036101499995311315, "prefetch": {"data_wait_seconds": 0.016062305001540267, "consumed_batches": 1707, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2257, "tokens": 168085966, "loss_tokens": 68974, "loss": 2.310276789079943, "lr": 0.0002994113505106179, "optimizer_lrs": [0.01996075670070786, 0.0002994113505106179], "gradient_norm": 0.5409759879112244, "seconds": 0.2935859320000418, "nonpadding_tokens_per_second": 234936.32521871035, "ranks": [{"compute_seconds": 0.28425876100004643, "nonpadding_tokens": 1927, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2878848849999258, "nonpadding_tokens": 4305, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2828996209999559, "nonpadding_tokens": 6972, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2910444130000087, "nonpadding_tokens": 14193, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28234379200000603, "nonpadding_tokens": 10699, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2852879389999998, "nonpadding_tokens": 15775, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2823039800000515, "nonpadding_tokens": 12785, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2910236500000565, "nonpadding_tokens": 2318, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003819979999661882, "prefetch": {"data_wait_seconds": 0.01607126300162065, "consumed_batches": 1708, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2258, "tokens": 168180429, "loss_tokens": 94463, "loss": 2.5330642023861194, "lr": 0.0002994105998274372, "optimizer_lrs": [0.01996070665516248, 0.0002994105998274372], "gradient_norm": 0.3667653799057007, "seconds": 0.28392227799997727, "nonpadding_tokens_per_second": 332707.2488478962, "ranks": [{"compute_seconds": 0.281472122000082, "nonpadding_tokens": 17381, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27812746800009336, "nonpadding_tokens": 4154, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27484936899998047, "nonpadding_tokens": 5029, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2802164920000223, "nonpadding_tokens": 11309, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2811412259999315, "nonpadding_tokens": 6093, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2766610870000932, "nonpadding_tokens": 20166, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2736998060000815, "nonpadding_tokens": 19261, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2742833059999157, "nonpadding_tokens": 11070, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003694370000175695, "prefetch": {"data_wait_seconds": 0.01607930900156873, "consumed_batches": 1709, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2259, "tokens": 168256714, "loss_tokens": 76285, "loss": 2.4458342891787375, "lr": 0.0002994099932537053, "optimizer_lrs": [0.019960666216913687, 0.0002994099932537053], "gradient_norm": 0.4317108988761902, "seconds": 0.2665157989999898, "nonpadding_tokens_per_second": 286230.6860840281, "ranks": [{"compute_seconds": 0.2576448290000144, "nonpadding_tokens": 5025, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25938959699999486, "nonpadding_tokens": 17017, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2617577319999782, "nonpadding_tokens": 11980, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2569454690000157, "nonpadding_tokens": 12625, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.262977836999994, "nonpadding_tokens": 3627, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25854984099999, "nonpadding_tokens": 4261, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.264032971000006, "nonpadding_tokens": 8698, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2561166509999566, "nonpadding_tokens": 13052, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040894599999319325, "prefetch": {"data_wait_seconds": 0.01609103600146966, "consumed_batches": 1710, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2260, "tokens": 168312535, "loss_tokens": 55821, "loss": 2.432994247236703, "lr": 0.000299409549200515, "optimizer_lrs": [0.019960636613367666, 0.000299409549200515], "gradient_norm": 0.6388000845909119, "seconds": 0.24632672699999603, "nonpadding_tokens_per_second": 226613.65528557077, "ranks": [{"compute_seconds": 0.23844731600001978, "nonpadding_tokens": 7679, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.241012678000061, "nonpadding_tokens": 3336, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23800550800001474, "nonpadding_tokens": 5584, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24217475000000377, "nonpadding_tokens": 4043, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23643560599998636, "nonpadding_tokens": 2922, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23967745800007378, "nonpadding_tokens": 12717, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24374532300009832, "nonpadding_tokens": 16412, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23742936099995404, "nonpadding_tokens": 3128, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003383499999927153, "prefetch": {"data_wait_seconds": 0.016099649001446892, "consumed_batches": 1711, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2261, "tokens": 168376750, "loss_tokens": 64215, "loss": 2.799345217238963, "lr": 0.0002994090381673327, "optimizer_lrs": [0.019960602544488847, 0.0002994090381673327], "gradient_norm": 0.4401805102825165, "seconds": 0.25640896100003374, "nonpadding_tokens_per_second": 250439.76524670504, "ranks": [{"compute_seconds": 0.25359362599999713, "nonpadding_tokens": 5628, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2508442549999472, "nonpadding_tokens": 4383, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25343873899998925, "nonpadding_tokens": 13582, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2488502730000164, "nonpadding_tokens": 10089, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2540198890000056, "nonpadding_tokens": 3331, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24935528299999987, "nonpadding_tokens": 9890, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24689609400002155, "nonpadding_tokens": 14804, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2471097820000523, "nonpadding_tokens": 2508, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033983100001933053, "prefetch": {"data_wait_seconds": 0.016107795001360614, "consumed_batches": 1712, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2262, "tokens": 168416558, "loss_tokens": 39808, "loss": 2.412834339203175, "lr": 0.00029940872125818895, "optimizer_lrs": [0.0199605814172126, 0.00029940872125818895], "gradient_norm": 0.4379461705684662, "seconds": 0.2948087119999627, "nonpadding_tokens_per_second": 135029.9308658322, "ranks": [{"compute_seconds": 0.286987364999959, "nonpadding_tokens": 2078, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2886287060000541, "nonpadding_tokens": 9831, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.286319029000083, "nonpadding_tokens": 10681, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29165865799996027, "nonpadding_tokens": 3282, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.292265911999948, "nonpadding_tokens": 1921, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2895324600000322, "nonpadding_tokens": 3522, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28551296299997375, "nonpadding_tokens": 3893, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2857020430000148, "nonpadding_tokens": 4600, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004254269999819371, "prefetch": {"data_wait_seconds": 0.016118020001457012, "consumed_batches": 1713, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2263, "tokens": 168458238, "loss_tokens": 41680, "loss": 2.2838634761876198, "lr": 0.00029940838935532704, "optimizer_lrs": [0.01996055929035514, 0.00029940838935532704], "gradient_norm": 0.3826499581336975, "seconds": 0.35513982299994495, "nonpadding_tokens_per_second": 117362.22552548397, "ranks": [{"compute_seconds": 0.35264552499995716, "nonpadding_tokens": 9662, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34862393400010205, "nonpadding_tokens": 6587, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34877181799993195, "nonpadding_tokens": 5384, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3490207009999722, "nonpadding_tokens": 3925, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3476203960000248, "nonpadding_tokens": 6888, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35076691899996604, "nonpadding_tokens": 4771, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35185523799998464, "nonpadding_tokens": 2661, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3479807259999461, "nonpadding_tokens": 1802, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003938739999966856, "prefetch": {"data_wait_seconds": 0.016126741001428968, "consumed_batches": 1714, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2264, "tokens": 168498275, "loss_tokens": 40037, "loss": 2.566393663985813, "lr": 0.00029940807044837145, "optimizer_lrs": [0.019960538029891433, 0.00029940807044837145], "gradient_norm": 0.4887874722480774, "seconds": 0.22629896000000826, "nonpadding_tokens_per_second": 176920.83074530496, "ranks": [{"compute_seconds": 0.2212027709999802, "nonpadding_tokens": 2716, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22139576999995825, "nonpadding_tokens": 2264, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21819591700000274, "nonpadding_tokens": 6038, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21953134799991858, "nonpadding_tokens": 9795, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2237585529999251, "nonpadding_tokens": 5284, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2209703340000715, "nonpadding_tokens": 1973, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21788303899995753, "nonpadding_tokens": 6937, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21820776499998829, "nonpadding_tokens": 5030, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004412400001001515, "prefetch": {"data_wait_seconds": 0.01613709200144342, "consumed_batches": 1715, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2265, "tokens": 168552252, "loss_tokens": 53977, "loss": 2.4441069807510605, "lr": 0.0002994076403693325, "optimizer_lrs": [0.0199605093579555, 0.0002994076403693325], "gradient_norm": 0.43369385600090027, "seconds": 0.2353159740000592, "nonpadding_tokens_per_second": 229380.94291884502, "ranks": [{"compute_seconds": 0.23004008899999917, "nonpadding_tokens": 5234, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2287709110000833, "nonpadding_tokens": 3118, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23007787899996401, "nonpadding_tokens": 8006, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22609951100002945, "nonpadding_tokens": 4551, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23205676399993536, "nonpadding_tokens": 12663, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22761860799994338, "nonpadding_tokens": 15301, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22513914499995735, "nonpadding_tokens": 2925, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23278124099999786, "nonpadding_tokens": 2179, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004375070000151027, "prefetch": {"data_wait_seconds": 0.016147174001460485, "consumed_batches": 1716, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2266, "tokens": 168615987, "loss_tokens": 63735, "loss": 2.5604742782615517, "lr": 0.00029940713233965504, "optimizer_lrs": [0.019960475489310337, 0.00029940713233965504], "gradient_norm": 0.3881555199623108, "seconds": 0.27908991700007846, "nonpadding_tokens_per_second": 228367.26129372162, "ranks": [{"compute_seconds": 0.2763933359999555, "nonpadding_tokens": 4633, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2719883380000283, "nonpadding_tokens": 9619, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2687584990000005, "nonpadding_tokens": 6396, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2748565580000104, "nonpadding_tokens": 3684, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27486323799996626, "nonpadding_tokens": 11591, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.270755859000019, "nonpadding_tokens": 11648, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2748074670000733, "nonpadding_tokens": 6212, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27529435799999646, "nonpadding_tokens": 9952, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003894139999829349, "prefetch": {"data_wait_seconds": 0.016155197001424426, "consumed_batches": 1717, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2267, "tokens": 168729420, "loss_tokens": 113433, "loss": 2.2113656850299295, "lr": 0.00029940622763131077, "optimizer_lrs": [0.01996041517542072, 0.00029940622763131077], "gradient_norm": 0.4764661192893982, "seconds": 0.28647521599998527, "nonpadding_tokens_per_second": 395960.9546118846, "ranks": [{"compute_seconds": 0.28167005000000245, "nonpadding_tokens": 3963, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27943955599994297, "nonpadding_tokens": 1747, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2812293450000425, "nonpadding_tokens": 16183, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27678337399993325, "nonpadding_tokens": 10344, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2755365670000174, "nonpadding_tokens": 19025, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2794823339999084, "nonpadding_tokens": 24614, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2821053080000411, "nonpadding_tokens": 20604, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28382511400002386, "nonpadding_tokens": 16953, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004526399999349451, "prefetch": {"data_wait_seconds": 0.016165580001441526, "consumed_batches": 1718, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2268, "tokens": 168776793, "loss_tokens": 47373, "loss": 2.893808181875752, "lr": 0.00029940584959442046, "optimizer_lrs": [0.019960389972961366, 0.00029940584959442046], "gradient_norm": 0.46042829751968384, "seconds": 0.2607978280000225, "nonpadding_tokens_per_second": 181646.45144205692, "ranks": [{"compute_seconds": 0.2534184889999551, "nonpadding_tokens": 6752, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2550437920000377, "nonpadding_tokens": 6213, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2517874809999512, "nonpadding_tokens": 5047, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25779363099991315, "nonpadding_tokens": 6958, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2509838899999295, "nonpadding_tokens": 15128, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25509603799991964, "nonpadding_tokens": 2278, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25755859000003056, "nonpadding_tokens": 2697, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2581650280000076, "nonpadding_tokens": 2300, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004022079999685957, "prefetch": {"data_wait_seconds": 0.016174766001498142, "consumed_batches": 1719, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2269, "tokens": 168813966, "loss_tokens": 37173, "loss": 2.3966960646840447, "lr": 0.00029940555286955313, "optimizer_lrs": [0.019960370191303544, 0.00029940555286955313], "gradient_norm": 0.44291889667510986, "seconds": 0.2257412149999709, "nonpadding_tokens_per_second": 164670.85994910053, "ranks": [{"compute_seconds": 0.21650347100001, "nonpadding_tokens": 7727, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21872267799994916, "nonpadding_tokens": 6194, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21624517700001888, "nonpadding_tokens": 5751, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21712151400004132, "nonpadding_tokens": 2486, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21492491499998323, "nonpadding_tokens": 1958, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21943075100000442, "nonpadding_tokens": 4106, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2156623790000367, "nonpadding_tokens": 5300, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2231316730000117, "nonpadding_tokens": 3651, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004004119999763134, "prefetch": {"data_wait_seconds": 0.016184321001517787, "consumed_batches": 1720, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2270, "tokens": 168863639, "loss_tokens": 49673, "loss": 2.2839649432287157, "lr": 0.00029940515625095796, "optimizer_lrs": [0.019960343750063864, 0.00029940515625095796], "gradient_norm": 0.3831489384174347, "seconds": 0.5256290170000284, "nonpadding_tokens_per_second": 94502.01262385276, "ranks": [{"compute_seconds": 0.5229541090000112, "nonpadding_tokens": 9937, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5188523660000328, "nonpadding_tokens": 8338, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.516820098999915, "nonpadding_tokens": 8945, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.5217378380000355, "nonpadding_tokens": 7483, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.5217862009999408, "nonpadding_tokens": 2948, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5188332809999565, "nonpadding_tokens": 2241, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.5164714030000823, "nonpadding_tokens": 6418, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.5170349400000305, "nonpadding_tokens": 3363, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005308319999812738, "prefetch": {"data_wait_seconds": 0.016192150001486993, "consumed_batches": 1721, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2271, "tokens": 168912622, "loss_tokens": 48983, "loss": 2.3186736980176796, "lr": 0.00029940476501249104, "optimizer_lrs": [0.019960317667499405, 0.00029940476501249104], "gradient_norm": 0.4066620171070099, "seconds": 0.3298968369999784, "nonpadding_tokens_per_second": 148479.75035299658, "ranks": [{"compute_seconds": 0.3271543880000536, "nonpadding_tokens": 14365, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3245795330000192, "nonpadding_tokens": 4927, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3217419270000619, "nonpadding_tokens": 5840, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3261954209999658, "nonpadding_tokens": 3380, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3209231969999564, "nonpadding_tokens": 2414, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.323432061999938, "nonpadding_tokens": 5877, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32093121999992036, "nonpadding_tokens": 7736, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32143454999993537, "nonpadding_tokens": 4444, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045830199996999, "prefetch": {"data_wait_seconds": 0.016203124001506808, "consumed_batches": 1722, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2272, "tokens": 168963586, "loss_tokens": 50964, "loss": 2.1446923625009813, "lr": 0.0002994043578150966, "optimizer_lrs": [0.01996029052100644, 0.0002994043578150966], "gradient_norm": 0.45991405844688416, "seconds": 0.2807853100000557, "nonpadding_tokens_per_second": 181505.22190776252, "ranks": [{"compute_seconds": 0.2727960580000399, "nonpadding_tokens": 5542, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27513797999995404, "nonpadding_tokens": 11169, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27698641200004204, "nonpadding_tokens": 10041, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27246840600003, "nonpadding_tokens": 2326, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27140911700007564, "nonpadding_tokens": 9712, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27513643600002524, "nonpadding_tokens": 4501, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27134910900008435, "nonpadding_tokens": 1971, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27829618600003414, "nonpadding_tokens": 5702, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040486699992925423, "prefetch": {"data_wait_seconds": 0.01621295200148154, "consumed_batches": 1723, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2273, "tokens": 169000103, "loss_tokens": 36517, "loss": 2.467751321302407, "lr": 0.00029940406596239003, "optimizer_lrs": [0.019960271064159336, 0.00029940406596239003], "gradient_norm": 0.8083614110946655, "seconds": 0.25785975699989194, "nonpadding_tokens_per_second": 141615.7388219958, "ranks": [{"compute_seconds": 0.25284941899997193, "nonpadding_tokens": 3866, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2515985989999763, "nonpadding_tokens": 10340, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25373009300005833, "nonpadding_tokens": 1741, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24923458099999607, "nonpadding_tokens": 3231, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25490557500006616, "nonpadding_tokens": 6018, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2517512040000156, "nonpadding_tokens": 4424, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24702940300005594, "nonpadding_tokens": 4738, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2553274030000239, "nonpadding_tokens": 2159, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039775400000507943, "prefetch": {"data_wait_seconds": 0.01622274200140339, "consumed_batches": 1724, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2274, "tokens": 169073596, "loss_tokens": 73493, "loss": 2.442855101846434, "lr": 0.0002994034783723151, "optimizer_lrs": [0.019960231891487674, 0.0002994034783723151], "gradient_norm": 0.4007674753665924, "seconds": 0.3061721870000156, "nonpadding_tokens_per_second": 240038.13252964176, "ranks": [{"compute_seconds": 0.2880952599999773, "nonpadding_tokens": 12567, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2907807739999271, "nonpadding_tokens": 14100, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2870520789999773, "nonpadding_tokens": 6325, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2874351469999965, "nonpadding_tokens": 3980, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29442822799990154, "nonpadding_tokens": 6970, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29211672799999633, "nonpadding_tokens": 11351, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29721421599992937, "nonpadding_tokens": 12892, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29519788699997207, "nonpadding_tokens": 5308, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034366300008059625, "prefetch": {"data_wait_seconds": 0.01623101600144139, "consumed_batches": 1725, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2275, "tokens": 169151311, "loss_tokens": 77715, "loss": 2.339076071221772, "lr": 0.0002994028567123035, "optimizer_lrs": [0.019960190447486904, 0.0002994028567123035], "gradient_norm": 0.37576794624328613, "seconds": 0.3084341559999757, "nonpadding_tokens_per_second": 251966.25758920852, "ranks": [{"compute_seconds": 0.3029690270000174, "nonpadding_tokens": 10278, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3024649459999864, "nonpadding_tokens": 9343, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30445292599995355, "nonpadding_tokens": 4384, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3051905870000837, "nonpadding_tokens": 15301, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3056669879999845, "nonpadding_tokens": 14036, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30181393800000933, "nonpadding_tokens": 5969, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3056307489999881, "nonpadding_tokens": 16570, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2988773009999477, "nonpadding_tokens": 1834, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047959599999103375, "prefetch": {"data_wait_seconds": 0.016242284001535268, "consumed_batches": 1726, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2276, "tokens": 169229706, "loss_tokens": 78395, "loss": 2.495987068690605, "lr": 0.0002994022292855224, "optimizer_lrs": [0.01996014861903483, 0.0002994022292855224], "gradient_norm": 0.34825220704078674, "seconds": 0.3466748570000391, "nonpadding_tokens_per_second": 226134.08044181054, "ranks": [{"compute_seconds": 0.33878256799994233, "nonpadding_tokens": 13769, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33952561199998854, "nonpadding_tokens": 19002, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34389965000002576, "nonpadding_tokens": 7181, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33764730300003976, "nonpadding_tokens": 5180, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33603479500004596, "nonpadding_tokens": 6091, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3403635799999165, "nonpadding_tokens": 6166, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33547228499992343, "nonpadding_tokens": 11019, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3360444699999334, "nonpadding_tokens": 9987, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035431199989943707, "prefetch": {"data_wait_seconds": 0.016251121001531033, "consumed_batches": 1727, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2277, "tokens": 169296749, "loss_tokens": 67043, "loss": 2.6951793065644436, "lr": 0.0002994016924526078, "optimizer_lrs": [0.019960112830173855, 0.0002994016924526078], "gradient_norm": 0.44059860706329346, "seconds": 0.4032857659999536, "nonpadding_tokens_per_second": 166241.91987973044, "ranks": [{"compute_seconds": 0.39454068899999584, "nonpadding_tokens": 11009, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.39630134699996233, "nonpadding_tokens": 12974, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3932862070000738, "nonpadding_tokens": 3645, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.39983755699995527, "nonpadding_tokens": 6431, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3921186929999294, "nonpadding_tokens": 7950, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39538576499990086, "nonpadding_tokens": 13848, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.39261604900002567, "nonpadding_tokens": 4287, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4008192769999823, "nonpadding_tokens": 6899, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003442060000224956, "prefetch": {"data_wait_seconds": 0.01625964000147917, "consumed_batches": 1728, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2278, "tokens": 169355056, "loss_tokens": 58307, "loss": 2.4623193291543037, "lr": 0.00029940122537593957, "optimizer_lrs": [0.019960081691729307, 0.00029940122537593957], "gradient_norm": 1.0742405652999878, "seconds": 0.3158284700000422, "nonpadding_tokens_per_second": 184616.03540678966, "ranks": [{"compute_seconds": 0.31289155799993296, "nonpadding_tokens": 5028, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3093248669999866, "nonpadding_tokens": 4245, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3071897470000522, "nonpadding_tokens": 11588, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30780097599995315, "nonpadding_tokens": 5730, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3062443170000506, "nonpadding_tokens": 5641, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31004388899998503, "nonpadding_tokens": 11053, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30628142199998365, "nonpadding_tokens": 5193, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31320456399998875, "nonpadding_tokens": 9829, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000397180000049957, "prefetch": {"data_wait_seconds": 0.01626850600150931, "consumed_batches": 1729, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2279, "tokens": 169412250, "loss_tokens": 57194, "loss": 2.7470858503514353, "lr": 0.00029940076703845403, "optimizer_lrs": [0.019960051135896936, 0.00029940076703845403], "gradient_norm": 0.6572823524475098, "seconds": 0.2772472620000599, "nonpadding_tokens_per_second": 206292.3889217259, "ranks": [{"compute_seconds": 0.273245807999956, "nonpadding_tokens": 6891, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2723902500000577, "nonpadding_tokens": 3874, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2689767619999657, "nonpadding_tokens": 5848, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26948215599998093, "nonpadding_tokens": 10818, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2676490510000349, "nonpadding_tokens": 3668, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2704865289999816, "nonpadding_tokens": 9880, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26744962199995825, "nonpadding_tokens": 12938, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27466984800003047, "nonpadding_tokens": 3277, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037061000000448985, "prefetch": {"data_wait_seconds": 0.016276930001552614, "consumed_batches": 1730, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2280, "tokens": 169499262, "loss_tokens": 87012, "loss": 2.347286868477911, "lr": 0.00029940006941189313, "optimizer_lrs": [0.019960004627459543, 0.00029940006941189313], "gradient_norm": 0.4191408157348633, "seconds": 0.2913740809999581, "nonpadding_tokens_per_second": 298626.4244965993, "ranks": [{"compute_seconds": 0.2837878489999639, "nonpadding_tokens": 5637, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28292924000004405, "nonpadding_tokens": 6982, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27994816799991895, "nonpadding_tokens": 13944, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2861293350000551, "nonpadding_tokens": 4591, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28869798400000946, "nonpadding_tokens": 9302, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2818290469999738, "nonpadding_tokens": 4324, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2790607709999904, "nonpadding_tokens": 21786, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27964837300010004, "nonpadding_tokens": 20446, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004236319999790794, "prefetch": {"data_wait_seconds": 0.01628497800159323, "consumed_batches": 1731, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2281, "tokens": 169591912, "loss_tokens": 92650, "loss": 2.6261978885590933, "lr": 0.00029939932613700255, "optimizer_lrs": [0.01995995507580017, 0.00029939932613700255], "gradient_norm": 0.4137711822986603, "seconds": 0.2807641650000505, "nonpadding_tokens_per_second": 329992.25524376775, "ranks": [{"compute_seconds": 0.2777426040000819, "nonpadding_tokens": 19206, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2754080079999994, "nonpadding_tokens": 13577, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27662615099995946, "nonpadding_tokens": 5121, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2731510129999606, "nonpadding_tokens": 11769, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27757953499997257, "nonpadding_tokens": 5576, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27396720100000493, "nonpadding_tokens": 5391, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2711846169999035, "nonpadding_tokens": 14376, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2783552649999592, "nonpadding_tokens": 17634, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004050920000508995, "prefetch": {"data_wait_seconds": 0.016296160001573412, "consumed_batches": 1732, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2282, "tokens": 169694110, "loss_tokens": 102198, "loss": 2.4009461107360224, "lr": 0.00029939850573175236, "optimizer_lrs": [0.019959900382116823, 0.00029939850573175236], "gradient_norm": 0.4267013669013977, "seconds": 0.27299657200001093, "nonpadding_tokens_per_second": 374356.34906066116, "ranks": [{"compute_seconds": 0.27073254999993424, "nonpadding_tokens": 7243, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2676472719999765, "nonpadding_tokens": 10272, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26443761499990615, "nonpadding_tokens": 11381, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2647446759999639, "nonpadding_tokens": 2994, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26396774399995593, "nonpadding_tokens": 20389, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26612787199997, "nonpadding_tokens": 17144, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2633097819999648, "nonpadding_tokens": 25733, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2698258689999875, "nonpadding_tokens": 7042, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034184399999048765, "prefetch": {"data_wait_seconds": 0.01630455100151096, "consumed_batches": 1733, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2283, "tokens": 169767532, "loss_tokens": 73422, "loss": 2.3093440147367272, "lr": 0.00029939791598405387, "optimizer_lrs": [0.019959861065603594, 0.00029939791598405387], "gradient_norm": 0.5913938283920288, "seconds": 0.2821759700000257, "nonpadding_tokens_per_second": 260199.335896651, "ranks": [{"compute_seconds": 0.2771587429999727, "nonpadding_tokens": 4070, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2767279619999954, "nonpadding_tokens": 4218, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.273777516999985, "nonpadding_tokens": 17529, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2792051599999468, "nonpadding_tokens": 4422, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27973122200000944, "nonpadding_tokens": 10456, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2771591380000018, "nonpadding_tokens": 5395, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27293980199999623, "nonpadding_tokens": 10344, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2734494370000675, "nonpadding_tokens": 16988, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037672799999199924, "prefetch": {"data_wait_seconds": 0.01631326800141153, "consumed_batches": 1734, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2284, "tokens": 169835203, "loss_tokens": 67671, "loss": 2.250002078068892, "lr": 0.00029939737217479003, "optimizer_lrs": [0.01995982481165267, 0.00029939737217479003], "gradient_norm": 0.42486050724983215, "seconds": 0.29738713500000813, "nonpadding_tokens_per_second": 227551.8744279175, "ranks": [{"compute_seconds": 0.290284918999987, "nonpadding_tokens": 5613, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2900077780000174, "nonpadding_tokens": 6229, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28734149799993247, "nonpadding_tokens": 4083, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.293056432999947, "nonpadding_tokens": 9595, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.294792689000019, "nonpadding_tokens": 1233, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2897990410000375, "nonpadding_tokens": 9644, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28635985699997946, "nonpadding_tokens": 10909, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2877049070000339, "nonpadding_tokens": 20365, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004080169999269856, "prefetch": {"data_wait_seconds": 0.016323188001436506, "consumed_batches": 1735, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2285, "tokens": 169883837, "loss_tokens": 48634, "loss": 2.4537611290455237, "lr": 0.0002993969811970294, "optimizer_lrs": [0.019959798746468627, 0.0002993969811970294], "gradient_norm": 0.408945232629776, "seconds": 0.26459501399995133, "nonpadding_tokens_per_second": 183805.42877504465, "ranks": [{"compute_seconds": 0.25919241600001897, "nonpadding_tokens": 6821, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.257447706999983, "nonpadding_tokens": 5627, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25828089599997384, "nonpadding_tokens": 4132, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2611187149999523, "nonpadding_tokens": 3933, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2621456639999451, "nonpadding_tokens": 5229, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2586868040000354, "nonpadding_tokens": 11268, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26147508099995775, "nonpadding_tokens": 4553, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25772244899997077, "nonpadding_tokens": 7071, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039357399998607434, "prefetch": {"data_wait_seconds": 0.016333276001390118, "consumed_batches": 1736, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2286, "tokens": 169934254, "loss_tokens": 50417, "loss": 2.519986128190888, "lr": 0.0002993965757518529, "optimizer_lrs": [0.019959771716790194, 0.0002993965757518529], "gradient_norm": 4.664082050323486, "seconds": 0.2727142090000143, "nonpadding_tokens_per_second": 184871.1887248873, "ranks": [{"compute_seconds": 0.2693075669999416, "nonpadding_tokens": 6557, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2702198680000265, "nonpadding_tokens": 10214, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2673063339999544, "nonpadding_tokens": 3063, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2699983219999922, "nonpadding_tokens": 2531, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2658093899999585, "nonpadding_tokens": 6229, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2674137259999725, "nonpadding_tokens": 6212, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26976358600006733, "nonpadding_tokens": 6583, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.266944814999988, "nonpadding_tokens": 9028, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003518320000921449, "prefetch": {"data_wait_seconds": 0.016342343001340964, "consumed_batches": 1737, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2287, "tokens": 170009989, "loss_tokens": 75735, "loss": 2.6136572010959265, "lr": 0.0002993959664480078, "optimizer_lrs": [0.019959731096533855, 0.0002993959664480078], "gradient_norm": 0.4112391471862793, "seconds": 0.23282910000000356, "nonpadding_tokens_per_second": 325281.5047603536, "ranks": [{"compute_seconds": 0.23025352499996643, "nonpadding_tokens": 7479, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2304840280000917, "nonpadding_tokens": 15204, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.226972131000025, "nonpadding_tokens": 10846, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22794099499992626, "nonpadding_tokens": 11434, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22645373300008487, "nonpadding_tokens": 6608, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2292101199999479, "nonpadding_tokens": 13892, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22630156000002444, "nonpadding_tokens": 3638, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2265866760000108, "nonpadding_tokens": 6634, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004349060000095051, "prefetch": {"data_wait_seconds": 0.016351627001384372, "consumed_batches": 1738, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2288, "tokens": 170086588, "loss_tokens": 76599, "loss": 2.2069752052898863, "lr": 0.00029939534988105787, "optimizer_lrs": [0.019959689992070526, 0.00029939534988105787], "gradient_norm": 0.4053124189376831, "seconds": 0.362541614999941, "nonpadding_tokens_per_second": 211283.33088054587, "ranks": [{"compute_seconds": 0.35811001499996564, "nonpadding_tokens": 3914, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3602527340000279, "nonpadding_tokens": 17437, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3596220509999739, "nonpadding_tokens": 8919, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3600237980000429, "nonpadding_tokens": 8323, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3565335440000581, "nonpadding_tokens": 18312, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3575906340000756, "nonpadding_tokens": 8680, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3564437219999945, "nonpadding_tokens": 9423, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35705312400000366, "nonpadding_tokens": 1591, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034989400000995374, "prefetch": {"data_wait_seconds": 0.016360155001464705, "consumed_batches": 1739, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2289, "tokens": 170131465, "loss_tokens": 44877, "loss": 2.077363717773024, "lr": 0.0002993949885076467, "optimizer_lrs": [0.019959665900509783, 0.0002993949885076467], "gradient_norm": 0.5187434554100037, "seconds": 0.18824569199989583, "nonpadding_tokens_per_second": 238395.8938089528, "ranks": [{"compute_seconds": 0.185823287000062, "nonpadding_tokens": 4874, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.18231162300003234, "nonpadding_tokens": 7623, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.18198280100000375, "nonpadding_tokens": 6082, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.1849881790000154, "nonpadding_tokens": 10731, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1809572949999847, "nonpadding_tokens": 3772, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1828614700000344, "nonpadding_tokens": 2851, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.18068432699999448, "nonpadding_tokens": 5719, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.18598451999991994, "nonpadding_tokens": 3225, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033512100003463274, "prefetch": {"data_wait_seconds": 0.016368140001375764, "consumed_batches": 1740, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2290, "tokens": 170188472, "loss_tokens": 57007, "loss": 2.4543660756573753, "lr": 0.0002993945293017203, "optimizer_lrs": [0.019959635286781355, 0.0002993945293017203], "gradient_norm": 0.47061264514923096, "seconds": 0.2538898479999716, "nonpadding_tokens_per_second": 224534.38154016453, "ranks": [{"compute_seconds": 0.2512358649998987, "nonpadding_tokens": 2714, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25145463199999085, "nonpadding_tokens": 14285, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24864758100000017, "nonpadding_tokens": 14932, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2510522410000249, "nonpadding_tokens": 5067, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2474332219999269, "nonpadding_tokens": 10024, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24886666800000512, "nonpadding_tokens": 4032, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24743481699999847, "nonpadding_tokens": 1384, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2515922469999623, "nonpadding_tokens": 4569, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003631200000882018, "prefetch": {"data_wait_seconds": 0.016376007001397852, "consumed_batches": 1741, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2291, "tokens": 170220140, "loss_tokens": 31668, "loss": 2.430995257436529, "lr": 0.00029939427413280676, "optimizer_lrs": [0.019959618275520452, 0.00029939427413280676], "gradient_norm": 0.4647060036659241, "seconds": 0.2134398110000575, "nonpadding_tokens_per_second": 148369.69659793912, "ranks": [{"compute_seconds": 0.21110055999997712, "nonpadding_tokens": 2496, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2069880410000451, "nonpadding_tokens": 6275, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20778426900005798, "nonpadding_tokens": 2344, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21027212599994982, "nonpadding_tokens": 3730, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20641188000001875, "nonpadding_tokens": 3571, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20968529300000682, "nonpadding_tokens": 4874, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20576916699997128, "nonpadding_tokens": 3786, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20703064799999993, "nonpadding_tokens": 4592, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004413199999362405, "prefetch": {"data_wait_seconds": 0.01638537700144127, "consumed_batches": 1742, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2292, "tokens": 170299630, "loss_tokens": 79490, "loss": 1.8915400128947037, "lr": 0.00029939363339579286, "optimizer_lrs": [0.019959575559719524, 0.00029939363339579286], "gradient_norm": 1.0661189556121826, "seconds": 0.27285619700001007, "nonpadding_tokens_per_second": 291325.6172077963, "ranks": [{"compute_seconds": 0.26895752200005063, "nonpadding_tokens": 3701, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2705346230000032, "nonpadding_tokens": 3579, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26736518999996406, "nonpadding_tokens": 3670, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2674047159999873, "nonpadding_tokens": 5574, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2661931689999619, "nonpadding_tokens": 1219, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2689900109999144, "nonpadding_tokens": 21244, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26985552300004656, "nonpadding_tokens": 19639, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2666830850000679, "nonpadding_tokens": 20864, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003949700000021039, "prefetch": {"data_wait_seconds": 0.016393450001444876, "consumed_batches": 1743, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2293, "tokens": 170413457, "loss_tokens": 113827, "loss": 1.955757284080227, "lr": 0.0002993927152935477, "optimizer_lrs": [0.01995951435290318, 0.0002993927152935477], "gradient_norm": 1.3853118419647217, "seconds": 0.2843000990000064, "nonpadding_tokens_per_second": 400376.2235763324, "ranks": [{"compute_seconds": 0.2819762530000389, "nonpadding_tokens": 10755, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2807372720000103, "nonpadding_tokens": 18789, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27832738600000084, "nonpadding_tokens": 4397, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27899284199997965, "nonpadding_tokens": 17770, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27750768399994286, "nonpadding_tokens": 12588, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2802057879999893, "nonpadding_tokens": 9953, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27738570300004994, "nonpadding_tokens": 16958, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2777869129999999, "nonpadding_tokens": 22617, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039281299996218877, "prefetch": {"data_wait_seconds": 0.01640343700144342, "consumed_batches": 1744, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2294, "tokens": 170504568, "loss_tokens": 91111, "loss": 1.9176221175269725, "lr": 0.00029939197991398116, "optimizer_lrs": [0.019959465327598746, 0.00029939197991398116], "gradient_norm": 0.6636046767234802, "seconds": 0.3181430580000324, "nonpadding_tokens_per_second": 286383.74375590094, "ranks": [{"compute_seconds": 0.3159175600000026, "nonpadding_tokens": 24605, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3124661400000832, "nonpadding_tokens": 20940, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3155658760000506, "nonpadding_tokens": 11058, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31359734199998, "nonpadding_tokens": 10498, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31150392300003205, "nonpadding_tokens": 14423, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31482677899998635, "nonpadding_tokens": 3305, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3156934309999997, "nonpadding_tokens": 1357, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3125256260000242, "nonpadding_tokens": 4925, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003557889999683539, "prefetch": {"data_wait_seconds": 0.01641272300146568, "consumed_batches": 1745, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2295, "tokens": 170555947, "loss_tokens": 51379, "loss": 2.4677520545845577, "lr": 0.0002993915650255054, "optimizer_lrs": [0.01995943766836703, 0.0002993915650255054], "gradient_norm": 0.48070117831230164, "seconds": 0.2321697419999964, "nonpadding_tokens_per_second": 221299.29403117826, "ranks": [{"compute_seconds": 0.22936882600004083, "nonpadding_tokens": 6124, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22590854199995647, "nonpadding_tokens": 4057, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22861249699997188, "nonpadding_tokens": 10950, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22990855699993062, "nonpadding_tokens": 8153, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22624681000002056, "nonpadding_tokens": 1080, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22870494500000405, "nonpadding_tokens": 4518, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2256648289999248, "nonpadding_tokens": 8735, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22623639599999024, "nonpadding_tokens": 7762, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003362710000374136, "prefetch": {"data_wait_seconds": 0.016420707001501, "consumed_batches": 1746, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2296, "tokens": 170611652, "loss_tokens": 55705, "loss": 2.60372161385872, "lr": 0.0002993911150448078, "optimizer_lrs": [0.019959407669653855, 0.0002993911150448078], "gradient_norm": 0.38674989342689514, "seconds": 0.23327752000000146, "nonpadding_tokens_per_second": 238792.83353149352, "ranks": [{"compute_seconds": 0.2287728750000042, "nonpadding_tokens": 4771, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2267885579999529, "nonpadding_tokens": 5333, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22988573899999665, "nonpadding_tokens": 4677, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2281517229999963, "nonpadding_tokens": 9066, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23074484299991127, "nonpadding_tokens": 12373, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22941304099992976, "nonpadding_tokens": 5807, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23053573400000005, "nonpadding_tokens": 5678, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23098345000005338, "nonpadding_tokens": 8000, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037827599999218364, "prefetch": {"data_wait_seconds": 0.016429011001449112, "consumed_batches": 1747, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2297, "tokens": 170654877, "loss_tokens": 43225, "loss": 2.2145317018507806, "lr": 0.0002993907657622363, "optimizer_lrs": [0.019959384384149088, 0.0002993907657622363], "gradient_norm": 0.42244940996170044, "seconds": 0.22813749600004485, "nonpadding_tokens_per_second": 189469.07350991308, "ranks": [{"compute_seconds": 0.19206428600000436, "nonpadding_tokens": 2355, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2042513809999491, "nonpadding_tokens": 3426, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20132441600003403, "nonpadding_tokens": 6258, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20171751099996982, "nonpadding_tokens": 10326, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20084442100005617, "nonpadding_tokens": 3816, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20161283100003402, "nonpadding_tokens": 2643, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20047639799997796, "nonpadding_tokens": 10177, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20090746899995793, "nonpadding_tokens": 4224, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.008282358999963435, "prefetch": {"data_wait_seconds": 0.01643703100144478, "consumed_batches": 1748, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2298, "tokens": 170698902, "loss_tokens": 44025, "loss": 2.61721465076661, "lr": 0.00029939040991249226, "optimizer_lrs": [0.019959360660832818, 0.00029939040991249226], "gradient_norm": 1.0237373113632202, "seconds": 0.37797834500008776, "nonpadding_tokens_per_second": 116474.92662572979, "ranks": [{"compute_seconds": 0.3730418179999333, "nonpadding_tokens": 6330, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3750907040000584, "nonpadding_tokens": 2493, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3723477799999273, "nonpadding_tokens": 7973, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3730634229999623, "nonpadding_tokens": 6097, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3709856140000056, "nonpadding_tokens": 3010, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37263812300000154, "nonpadding_tokens": 2274, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3709203970000772, "nonpadding_tokens": 14495, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37561617500000466, "nonpadding_tokens": 1353, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000368129999969824, "prefetch": {"data_wait_seconds": 0.01645541800139938, "consumed_batches": 1749, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2299, "tokens": 170748788, "loss_tokens": 49886, "loss": 2.3697998624363548, "lr": 0.00029939000656358395, "optimizer_lrs": [0.0199593337709056, 0.00029939000656358395], "gradient_norm": 0.3918532133102417, "seconds": 0.29236288900006, "nonpadding_tokens_per_second": 170630.41130363764, "ranks": [{"compute_seconds": 0.28855497099993954, "nonpadding_tokens": 1681, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2899209319999727, "nonpadding_tokens": 5101, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28746508199992604, "nonpadding_tokens": 10737, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2879975190001005, "nonpadding_tokens": 3736, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2855669420000595, "nonpadding_tokens": 6252, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28920750299994324, "nonpadding_tokens": 4814, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2853805190000003, "nonpadding_tokens": 6105, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28782414700003756, "nonpadding_tokens": 11460, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034283399998003006, "prefetch": {"data_wait_seconds": 0.016463642001397716, "consumed_batches": 1750, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2300, "tokens": 170809683, "loss_tokens": 60895, "loss": 2.504922150833402, "lr": 0.0002993895140219698, "optimizer_lrs": [0.01995930093479799, 0.0002993895140219698], "gradient_norm": 0.3723161518573761, "seconds": 0.2200583450000977, "nonpadding_tokens_per_second": 276722.066595443, "ranks": [{"compute_seconds": 0.21461659600004168, "nonpadding_tokens": 1831, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21659460799992303, "nonpadding_tokens": 2552, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21600381400003243, "nonpadding_tokens": 6352, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21468218999996225, "nonpadding_tokens": 4559, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21690740199994707, "nonpadding_tokens": 14103, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21558932400000685, "nonpadding_tokens": 12747, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21279233100005968, "nonpadding_tokens": 10569, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21763762199998382, "nonpadding_tokens": 8182, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039622099995995086, "prefetch": {"data_wait_seconds": 0.01647380900131168, "consumed_batches": 1751, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2301, "tokens": 170888308, "loss_tokens": 78625, "loss": 2.5169095786963434, "lr": 0.00029938887778012, "optimizer_lrs": [0.01995925851867467, 0.00029938887778012], "gradient_norm": 0.4862388074398041, "seconds": 0.31753490400001283, "nonpadding_tokens_per_second": 247610.57448977898, "ranks": [{"compute_seconds": 0.31104298499997185, "nonpadding_tokens": 9266, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31087822699998924, "nonpadding_tokens": 20878, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31361383899991324, "nonpadding_tokens": 3755, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3113225849999708, "nonpadding_tokens": 10584, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3143141169999808, "nonpadding_tokens": 9105, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3119228570000132, "nonpadding_tokens": 10480, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3094058489999725, "nonpadding_tokens": 12416, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3149358820000998, "nonpadding_tokens": 2141, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034557599997242505, "prefetch": {"data_wait_seconds": 0.01648217600127566, "consumed_batches": 1752, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2302, "tokens": 170985133, "loss_tokens": 96825, "loss": 2.41506567906016, "lr": 0.0002993880938076664, "optimizer_lrs": [0.019959206253844425, 0.0002993880938076664], "gradient_norm": 0.41019147634506226, "seconds": 0.3527404179999394, "nonpadding_tokens_per_second": 274493.63628076395, "ranks": [{"compute_seconds": 0.34919433000004574, "nonpadding_tokens": 12044, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34729566100008924, "nonpadding_tokens": 2907, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35013613400008126, "nonpadding_tokens": 14204, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.348264709000091, "nonpadding_tokens": 19975, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3471698959999685, "nonpadding_tokens": 7659, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34864566300007027, "nonpadding_tokens": 4982, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.347337648000007, "nonpadding_tokens": 12042, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3478556820000449, "nonpadding_tokens": 23012, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038689600000907376, "prefetch": {"data_wait_seconds": 0.016491163001276732, "consumed_batches": 1753, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2303, "tokens": 171070377, "loss_tokens": 85244, "loss": 2.4357819538618553, "lr": 0.00029938740318920617, "optimizer_lrs": [0.019959160212613746, 0.00029938740318920617], "gradient_norm": 0.4081963300704956, "seconds": 0.3945316089999551, "nonpadding_tokens_per_second": 216063.80339479898, "ranks": [{"compute_seconds": 0.39217365499996504, "nonpadding_tokens": 24255, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.38865832900000896, "nonpadding_tokens": 7589, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3896581139999853, "nonpadding_tokens": 4956, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.38942450399997597, "nonpadding_tokens": 6339, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.38830543699998543, "nonpadding_tokens": 5173, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39136055899996336, "nonpadding_tokens": 13387, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.38820313700000497, "nonpadding_tokens": 10017, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.38894052399996326, "nonpadding_tokens": 13528, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003946179999729793, "prefetch": {"data_wait_seconds": 0.016498524001193005, "consumed_batches": 1754, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2304, "tokens": 171120821, "loss_tokens": 50444, "loss": 2.3023637598128617, "lr": 0.00029938699432571124, "optimizer_lrs": [0.019959132955047417, 0.00029938699432571124], "gradient_norm": 0.42180848121643066, "seconds": 0.23535050300006333, "nonpadding_tokens_per_second": 214335.6370901252, "ranks": [{"compute_seconds": 0.23293233100002908, "nonpadding_tokens": 5874, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22886689899996782, "nonpadding_tokens": 5418, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22605809799995313, "nonpadding_tokens": 6590, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23237097499998072, "nonpadding_tokens": 4212, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22500241900002038, "nonpadding_tokens": 14047, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2281180369999447, "nonpadding_tokens": 5875, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23265111100010927, "nonpadding_tokens": 2554, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22505738499990002, "nonpadding_tokens": 5874, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003751409999495081, "prefetch": {"data_wait_seconds": 0.016508019001165053, "consumed_batches": 1755, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2305, "tokens": 171191689, "loss_tokens": 70868, "loss": 2.5547893619122877, "lr": 0.00029938641968978256, "optimizer_lrs": [0.019959094645985505, 0.00029938641968978256], "gradient_norm": 0.4618481993675232, "seconds": 0.26924093299999186, "nonpadding_tokens_per_second": 263214.06336829974, "ranks": [{"compute_seconds": 0.26630883699999686, "nonpadding_tokens": 6701, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.266936100999942, "nonpadding_tokens": 16700, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26591912499998216, "nonpadding_tokens": 6415, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2660988009999983, "nonpadding_tokens": 7404, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26302492800004984, "nonpadding_tokens": 10482, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26388992299996517, "nonpadding_tokens": 8543, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2666611480000256, "nonpadding_tokens": 9571, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26327052399994955, "nonpadding_tokens": 5052, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003856710000036401, "prefetch": {"data_wait_seconds": 0.01651786000122968, "consumed_batches": 1756, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2306, "tokens": 171291490, "loss_tokens": 99801, "loss": 2.424630170288875, "lr": 0.00029938560999403466, "optimizer_lrs": [0.01995904066626898, 0.00029938560999403466], "gradient_norm": 0.5547146201133728, "seconds": 0.30574319999993804, "nonpadding_tokens_per_second": 326420.9964441408, "ranks": [{"compute_seconds": 0.2979230629999847, "nonpadding_tokens": 7545, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3002205459999914, "nonpadding_tokens": 27218, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29662616699999944, "nonpadding_tokens": 19131, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29719877099989844, "nonpadding_tokens": 12344, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2961183099999971, "nonpadding_tokens": 9352, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29878323699995235, "nonpadding_tokens": 4867, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2959137479999754, "nonpadding_tokens": 7685, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3034214609999708, "nonpadding_tokens": 11659, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003764429999364438, "prefetch": {"data_wait_seconds": 0.016526142001183075, "consumed_batches": 1757, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2307, "tokens": 171344316, "loss_tokens": 52826, "loss": 2.1974894583159807, "lr": 0.0002993851811956819, "optimizer_lrs": [0.019959012079712127, 0.0002993851811956819], "gradient_norm": 0.46909698843955994, "seconds": 0.3147370200000523, "nonpadding_tokens_per_second": 167841.71115298485, "ranks": [{"compute_seconds": 0.3122808050000003, "nonpadding_tokens": 14695, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30866412600005333, "nonpadding_tokens": 1906, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31194308900001033, "nonpadding_tokens": 9576, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3099379269999645, "nonpadding_tokens": 6756, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3087077300000374, "nonpadding_tokens": 2865, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31131227299999864, "nonpadding_tokens": 7244, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3082705839999562, "nonpadding_tokens": 2645, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30888560300002155, "nonpadding_tokens": 7139, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003394029999981285, "prefetch": {"data_wait_seconds": 0.016535305001141865, "consumed_batches": 1758, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2308, "tokens": 171391289, "loss_tokens": 46973, "loss": 2.387528307485151, "lr": 0.0002993847997818524, "optimizer_lrs": [0.019958986652123492, 0.0002993847997818524], "gradient_norm": 0.6427449584007263, "seconds": 0.26660682700003235, "nonpadding_tokens_per_second": 176188.28643121844, "ranks": [{"compute_seconds": 0.26424629100006314, "nonpadding_tokens": 3581, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2637573340000472, "nonpadding_tokens": 2314, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26086101700002473, "nonpadding_tokens": 14787, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26155898299998626, "nonpadding_tokens": 3051, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2639210690000482, "nonpadding_tokens": 10618, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26280670799997097, "nonpadding_tokens": 2284, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25949193999997533, "nonpadding_tokens": 7812, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26415610499998365, "nonpadding_tokens": 2526, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034560400001737435, "prefetch": {"data_wait_seconds": 0.01654368700117459, "consumed_batches": 1759, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2309, "tokens": 171469984, "loss_tokens": 78695, "loss": 2.1566566332041424, "lr": 0.00029938416052561974, "optimizer_lrs": [0.019958944035041316, 0.00029938416052561974], "gradient_norm": 0.43329617381095886, "seconds": 0.3111100600000327, "nonpadding_tokens_per_second": 252949.06889218473, "ranks": [{"compute_seconds": 0.3082952690000411, "nonpadding_tokens": 2315, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3069930159999785, "nonpadding_tokens": 17290, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3043471009999621, "nonpadding_tokens": 4733, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3052639249999629, "nonpadding_tokens": 3570, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30368740200003685, "nonpadding_tokens": 11462, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30654940900001293, "nonpadding_tokens": 25427, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30876504599996224, "nonpadding_tokens": 2750, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.30368839499999467, "nonpadding_tokens": 11148, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036691299999347393, "prefetch": {"data_wait_seconds": 0.01655295500115983, "consumed_batches": 1760, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2310, "tokens": 171535580, "loss_tokens": 65596, "loss": 2.3164569867065063, "lr": 0.00029938362742226135, "optimizer_lrs": [0.019958908494817424, 0.00029938362742226135], "gradient_norm": 0.389167845249176, "seconds": 0.29836267600001065, "nonpadding_tokens_per_second": 219853.23660254898, "ranks": [{"compute_seconds": 0.2960599819999743, "nonpadding_tokens": 8347, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2953920329999846, "nonpadding_tokens": 8212, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29240860099992005, "nonpadding_tokens": 2455, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29298018800000136, "nonpadding_tokens": 7514, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2926583290000053, "nonpadding_tokens": 10609, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2939270150000084, "nonpadding_tokens": 5123, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2954410549999693, "nonpadding_tokens": 10347, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2923188490000257, "nonpadding_tokens": 12989, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003911399999196874, "prefetch": {"data_wait_seconds": 0.01656281200105525, "consumed_batches": 1761, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2311, "tokens": 171602660, "loss_tokens": 67080, "loss": 2.1137580594066785, "lr": 0.00029938308202034805, "optimizer_lrs": [0.019958872134689873, 0.00029938308202034805], "gradient_norm": 0.4058684706687927, "seconds": 0.2959189339999284, "nonpadding_tokens_per_second": 226683.70385524648, "ranks": [{"compute_seconds": 0.29309421599998586, "nonpadding_tokens": 12780, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2894044600000143, "nonpadding_tokens": 12053, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28944327300007444, "nonpadding_tokens": 6111, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29015297299997656, "nonpadding_tokens": 10661, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29313641499993537, "nonpadding_tokens": 12632, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29006487799995284, "nonpadding_tokens": 4095, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2889025509999783, "nonpadding_tokens": 6662, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2935999459999721, "nonpadding_tokens": 2086, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038256499999533844, "prefetch": {"data_wait_seconds": 0.016580381001020328, "consumed_batches": 1762, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2312, "tokens": 171681108, "loss_tokens": 78448, "loss": 2.3324917461248216, "lr": 0.0002993824438842896, "optimizer_lrs": [0.019958829592285975, 0.0002993824438842896], "gradient_norm": 0.3641757071018219, "seconds": 0.3302594890000137, "nonpadding_tokens_per_second": 237534.43159962242, "ranks": [{"compute_seconds": 0.32311236300006385, "nonpadding_tokens": 3643, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32518550200006757, "nonpadding_tokens": 5050, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32652234199997565, "nonpadding_tokens": 27731, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.322606622999956, "nonpadding_tokens": 11107, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32137285999999676, "nonpadding_tokens": 8237, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32363131600004635, "nonpadding_tokens": 7932, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32756285000004937, "nonpadding_tokens": 11547, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3277656870000101, "nonpadding_tokens": 3201, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004058560000430589, "prefetch": {"data_wait_seconds": 0.016590613001042698, "consumed_batches": 1763, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2313, "tokens": 171752922, "loss_tokens": 71814, "loss": 2.4728641438298937, "lr": 0.00029938185942402813, "optimizer_lrs": [0.019958790628268542, 0.00029938185942402813], "gradient_norm": 0.4017329812049866, "seconds": 0.2823190210000348, "nonpadding_tokens_per_second": 254371.80869223524, "ranks": [{"compute_seconds": 0.2752702700000782, "nonpadding_tokens": 10909, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2760896639999828, "nonpadding_tokens": 4539, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.278274582999984, "nonpadding_tokens": 11504, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27559813200002736, "nonpadding_tokens": 3321, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27979300100003, "nonpadding_tokens": 9835, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27614109300009204, "nonpadding_tokens": 12946, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2734497569999803, "nonpadding_tokens": 12655, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27935326600004373, "nonpadding_tokens": 6105, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034874499999659747, "prefetch": {"data_wait_seconds": 0.016600055001006808, "consumed_batches": 1764, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2314, "tokens": 171818804, "loss_tokens": 65882, "loss": 2.174988521143863, "lr": 0.00029938132299894415, "optimizer_lrs": [0.019958754866596276, 0.00029938132299894415], "gradient_norm": 0.41673147678375244, "seconds": 0.28832449500009716, "nonpadding_tokens_per_second": 228499.48978486133, "ranks": [{"compute_seconds": 0.2860407320000604, "nonpadding_tokens": 7705, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2854367859999911, "nonpadding_tokens": 9974, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28295297900001515, "nonpadding_tokens": 1720, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2852314290000777, "nonpadding_tokens": 3600, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28164637499992295, "nonpadding_tokens": 10527, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28392738900004133, "nonpadding_tokens": 18302, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2821970640000018, "nonpadding_tokens": 11896, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2824950390000822, "nonpadding_tokens": 2158, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003804149999950823, "prefetch": {"data_wait_seconds": 0.016608746001068653, "consumed_batches": 1765, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2315, "tokens": 171907402, "loss_tokens": 88598, "loss": 2.3603211627237637, "lr": 0.00029938060124944576, "optimizer_lrs": [0.019958706749963053, 0.00029938060124944576], "gradient_norm": 0.45638346672058105, "seconds": 0.33191240299993297, "nonpadding_tokens_per_second": 266931.87479353667, "ranks": [{"compute_seconds": 0.32892140199999176, "nonpadding_tokens": 10007, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32960704599997825, "nonpadding_tokens": 20130, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3263788440000326, "nonpadding_tokens": 13137, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32790539200004787, "nonpadding_tokens": 4005, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3257910649999758, "nonpadding_tokens": 4019, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.328370681000024, "nonpadding_tokens": 9636, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3263979719999952, "nonpadding_tokens": 15934, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3261892609999677, "nonpadding_tokens": 11730, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038189500003227295, "prefetch": {"data_wait_seconds": 0.016617345001080253, "consumed_batches": 1766, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2316, "tokens": 171990747, "loss_tokens": 83345, "loss": 2.4541296718459416, "lr": 0.0002993799219094875, "optimizer_lrs": [0.019958661460632503, 0.0002993799219094875], "gradient_norm": 0.4734254479408264, "seconds": 0.3056211870000425, "nonpadding_tokens_per_second": 272706.8787936761, "ranks": [{"compute_seconds": 0.2992637410000043, "nonpadding_tokens": 10281, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29800489600006586, "nonpadding_tokens": 13335, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2966911749999781, "nonpadding_tokens": 9862, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29718686700005037, "nonpadding_tokens": 4607, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.295944564000024, "nonpadding_tokens": 10381, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29930959900002563, "nonpadding_tokens": 1441, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3029589150000902, "nonpadding_tokens": 7554, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3025060120000944, "nonpadding_tokens": 25884, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004744270000855977, "prefetch": {"data_wait_seconds": 0.016628427001094792, "consumed_batches": 1767, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2317, "tokens": 172057482, "loss_tokens": 66735, "loss": 2.6198686970854874, "lr": 0.00029937937768880846, "optimizer_lrs": [0.019958625179253897, 0.00029937937768880846], "gradient_norm": 0.44493159651756287, "seconds": 0.4092941950000295, "nonpadding_tokens_per_second": 163048.97752091303, "ranks": [{"compute_seconds": 0.4048224030000256, "nonpadding_tokens": 20354, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.40364258700003575, "nonpadding_tokens": 10469, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4031691120000005, "nonpadding_tokens": 3390, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4040380000000141, "nonpadding_tokens": 5593, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.40229991000001064, "nonpadding_tokens": 7801, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.40460822400007146, "nonpadding_tokens": 11639, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.40206753300003584, "nonpadding_tokens": 2813, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4068155549999801, "nonpadding_tokens": 4676, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003366250000453874, "prefetch": {"data_wait_seconds": 0.01663656300111427, "consumed_batches": 1768, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2318, "tokens": 172106113, "loss_tokens": 48631, "loss": 2.3280859624005266, "lr": 0.0002993789809553537, "optimizer_lrs": [0.019958598730356915, 0.0002993789809553537], "gradient_norm": 0.4408961832523346, "seconds": 0.32083767100004934, "nonpadding_tokens_per_second": 151575.0935618546, "ranks": [{"compute_seconds": 0.31844256799990944, "nonpadding_tokens": 11444, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3164432050000414, "nonpadding_tokens": 8454, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3174334509999426, "nonpadding_tokens": 4754, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3132863329999509, "nonpadding_tokens": 7498, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31200062499999603, "nonpadding_tokens": 1537, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3143957660000751, "nonpadding_tokens": 2641, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3113158529999964, "nonpadding_tokens": 2405, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.31239863400003287, "nonpadding_tokens": 9898, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035359200001039426, "prefetch": {"data_wait_seconds": 0.01664465400115489, "consumed_batches": 1769, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2319, "tokens": 172176717, "loss_tokens": 70604, "loss": 2.2617871331652597, "lr": 0.0002993784047402714, "optimizer_lrs": [0.019958560316018094, 0.0002993784047402714], "gradient_norm": 0.37456396222114563, "seconds": 0.2797097659999963, "nonpadding_tokens_per_second": 252418.7875513826, "ranks": [{"compute_seconds": 0.2239192939999839, "nonpadding_tokens": 3723, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2301812879999261, "nonpadding_tokens": 5288, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23168913200004226, "nonpadding_tokens": 2989, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22760845599998447, "nonpadding_tokens": 13767, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23270852999996805, "nonpadding_tokens": 13734, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22903589899999588, "nonpadding_tokens": 17579, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2262250240000867, "nonpadding_tokens": 5534, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22649763000003986, "nonpadding_tokens": 7990, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.023825093000027664, "prefetch": {"data_wait_seconds": 0.016654824001150814, "consumed_batches": 1770, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2320, "tokens": 172219826, "loss_tokens": 43109, "loss": 2.1487908919831127, "lr": 0.0002993780527869773, "optimizer_lrs": [0.019958536852465156, 0.0002993780527869773], "gradient_norm": 0.41922497749328613, "seconds": 0.23279669699991246, "nonpadding_tokens_per_second": 185178.7441813069, "ranks": [{"compute_seconds": 0.22868869000001268, "nonpadding_tokens": 1653, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2266414949999671, "nonpadding_tokens": 2926, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22436451600003693, "nonpadding_tokens": 8800, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2248173070000803, "nonpadding_tokens": 3301, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23008275199993022, "nonpadding_tokens": 3227, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22577183900000364, "nonpadding_tokens": 7837, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22310301599998184, "nonpadding_tokens": 8506, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23027358299998468, "nonpadding_tokens": 6859, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005215359999510838, "prefetch": {"data_wait_seconds": 0.016682129001154067, "consumed_batches": 1771, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2321, "tokens": 172301901, "loss_tokens": 82075, "loss": 2.364074398416083, "lr": 0.0002993773824302129, "optimizer_lrs": [0.019958492162014195, 0.0002993773824302129], "gradient_norm": 0.3954813778400421, "seconds": 0.28530557500005216, "nonpadding_tokens_per_second": 287674.0140811654, "ranks": [{"compute_seconds": 0.27938711899992086, "nonpadding_tokens": 12507, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27867865600001096, "nonpadding_tokens": 7590, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2777429289999418, "nonpadding_tokens": 6888, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27863069299996823, "nonpadding_tokens": 21766, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2812032369999997, "nonpadding_tokens": 3073, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2792066789999126, "nonpadding_tokens": 4477, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2810373269999218, "nonpadding_tokens": 16459, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28283511300003283, "nonpadding_tokens": 9315, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004037219999872832, "prefetch": {"data_wait_seconds": 0.016692276001052164, "consumed_batches": 1772, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2322, "tokens": 172375162, "loss_tokens": 73261, "loss": 1.9051445772648476, "lr": 0.0002993767837584955, "optimizer_lrs": [0.019958452250566368, 0.0002993767837584955], "gradient_norm": 0.54865962266922, "seconds": 0.38511592600002587, "nonpadding_tokens_per_second": 190231.0318893306, "ranks": [{"compute_seconds": 0.3819361939999908, "nonpadding_tokens": 4654, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3789573980000114, "nonpadding_tokens": 7542, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3772988909998958, "nonpadding_tokens": 7971, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3826588220000531, "nonpadding_tokens": 1762, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.37593058800007384, "nonpadding_tokens": 6682, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38174082399996223, "nonpadding_tokens": 16796, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37594467999997505, "nonpadding_tokens": 2815, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37708736399997633, "nonpadding_tokens": 25039, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003419700000222292, "prefetch": {"data_wait_seconds": 0.01670062100113512, "consumed_batches": 1773, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2323, "tokens": 172458784, "loss_tokens": 83622, "loss": 2.079443059242783, "lr": 0.00029937610006833764, "optimizer_lrs": [0.01995840667122251, 0.00029937610006833764], "gradient_norm": 0.8148088455200195, "seconds": 0.3502922559999888, "nonpadding_tokens_per_second": 238720.66415308556, "ranks": [{"compute_seconds": 0.3456523280000283, "nonpadding_tokens": 24486, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3444160640000291, "nonpadding_tokens": 6099, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3459673349999548, "nonpadding_tokens": 8563, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3475580220000438, "nonpadding_tokens": 8467, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34016293599995606, "nonpadding_tokens": 4820, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34333869399995365, "nonpadding_tokens": 4317, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3465789979999272, "nonpadding_tokens": 16493, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3470000639999853, "nonpadding_tokens": 10377, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003653210000038598, "prefetch": {"data_wait_seconds": 0.016709915001072773, "consumed_batches": 1774, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2324, "tokens": 172498059, "loss_tokens": 39275, "loss": 2.2627062778485043, "lr": 0.00029937577882843436, "optimizer_lrs": [0.019958385255228958, 0.00029937577882843436], "gradient_norm": 0.5303135514259338, "seconds": 0.25500077500009866, "nonpadding_tokens_per_second": 154019.13974569217, "ranks": [{"compute_seconds": 0.24729827999999543, "nonpadding_tokens": 2248, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2484847969999464, "nonpadding_tokens": 8936, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.250036295999962, "nonpadding_tokens": 1884, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25116214799993486, "nonpadding_tokens": 3587, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25253230700002405, "nonpadding_tokens": 5617, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24723146300004828, "nonpadding_tokens": 9021, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24455962000001819, "nonpadding_tokens": 4258, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25148290199990697, "nonpadding_tokens": 3724, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003476419999515201, "prefetch": {"data_wait_seconds": 0.01671822300109227, "consumed_batches": 1775, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2325, "tokens": 172570249, "loss_tokens": 72190, "loss": 2.4845152548829477, "lr": 0.0002993751881534822, "optimizer_lrs": [0.019958345876898816, 0.0002993751881534822], "gradient_norm": 0.4373432397842407, "seconds": 0.26259719699999096, "nonpadding_tokens_per_second": 274907.73254522774, "ranks": [{"compute_seconds": 0.25531507999994574, "nonpadding_tokens": 3437, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2566909970000779, "nonpadding_tokens": 4358, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25824940899997273, "nonpadding_tokens": 9676, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25402233299996624, "nonpadding_tokens": 2781, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2592653450000171, "nonpadding_tokens": 24134, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2551854949999779, "nonpadding_tokens": 13447, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2590314289999469, "nonpadding_tokens": 3464, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2600850869999931, "nonpadding_tokens": 10893, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003413030000274375, "prefetch": {"data_wait_seconds": 0.016726127001177815, "consumed_batches": 1776, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2326, "tokens": 172680127, "loss_tokens": 109878, "loss": 2.0680700981998217, "lr": 0.00029937428857186463, "optimizer_lrs": [0.019958285904790977, 0.00029937428857186463], "gradient_norm": 0.4640946686267853, "seconds": 0.29039628399993944, "nonpadding_tokens_per_second": 378372.6103052438, "ranks": [{"compute_seconds": 0.2839536800000815, "nonpadding_tokens": 11471, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28400251199991544, "nonpadding_tokens": 18376, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28287980200002494, "nonpadding_tokens": 22723, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2872954670000354, "nonpadding_tokens": 12120, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28801373900000726, "nonpadding_tokens": 9049, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28558992199998556, "nonpadding_tokens": 11689, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28184196099994097, "nonpadding_tokens": 16040, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2821608159999869, "nonpadding_tokens": 8410, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035963200002697704, "prefetch": {"data_wait_seconds": 0.01673463900119998, "consumed_batches": 1777, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2327, "tokens": 172758813, "loss_tokens": 78686, "loss": 2.3458119932389496, "lr": 0.0002993736439653927, "optimizer_lrs": [0.01995824293102618, 0.0002993736439653927], "gradient_norm": 0.42949315905570984, "seconds": 0.30006683300007353, "nonpadding_tokens_per_second": 262228.2483315333, "ranks": [{"compute_seconds": 0.29589426100005767, "nonpadding_tokens": 8405, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2952425080000012, "nonpadding_tokens": 9731, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29229639999994106, "nonpadding_tokens": 19993, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2976264709999441, "nonpadding_tokens": 16317, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2918045799999618, "nonpadding_tokens": 4090, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.294288146000099, "nonpadding_tokens": 9893, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29116077900005166, "nonpadding_tokens": 7597, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29213405200005127, "nonpadding_tokens": 2660, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003573940000478615, "prefetch": {"data_wait_seconds": 0.01674309200127766, "consumed_batches": 1778, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2328, "tokens": 172828450, "loss_tokens": 69637, "loss": 2.2459692763904244, "lr": 0.0002993730732133978, "optimizer_lrs": [0.01995820488089319, 0.0002993730732133978], "gradient_norm": 0.373823881149292, "seconds": 0.28704297400008727, "nonpadding_tokens_per_second": 242601.30470909498, "ranks": [{"compute_seconds": 0.2803155899999865, "nonpadding_tokens": 6249, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2803913929999453, "nonpadding_tokens": 11269, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28459644000008666, "nonpadding_tokens": 11879, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2803148460000102, "nonpadding_tokens": 9075, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28422775299998193, "nonpadding_tokens": 4572, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28108832100008385, "nonpadding_tokens": 9733, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.278262072000075, "nonpadding_tokens": 8495, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2795024369999055, "nonpadding_tokens": 8365, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003811340000083874, "prefetch": {"data_wait_seconds": 0.01675316600126564, "consumed_batches": 1779, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2329, "tokens": 172901303, "loss_tokens": 72853, "loss": 2.3574764851824908, "lr": 0.00029937247582517826, "optimizer_lrs": [0.019958165055011884, 0.00029937247582517826], "gradient_norm": 0.364490807056427, "seconds": 0.3912051329999713, "nonpadding_tokens_per_second": 186227.1065855502, "ranks": [{"compute_seconds": 0.3865880749999633, "nonpadding_tokens": 9762, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.388547524000046, "nonpadding_tokens": 3893, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3852733950000129, "nonpadding_tokens": 3201, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.38558341199995994, "nonpadding_tokens": 6280, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.38852986699998837, "nonpadding_tokens": 18683, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3851027599999952, "nonpadding_tokens": 5563, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3835586369999646, "nonpadding_tokens": 5907, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3849032089999582, "nonpadding_tokens": 19564, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039501900005234347, "prefetch": {"data_wait_seconds": 0.016761366001290412, "consumed_batches": 1780, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2330, "tokens": 172964381, "loss_tokens": 63078, "loss": 2.188139337407654, "lr": 0.0002993719583618481, "optimizer_lrs": [0.01995813055745654, 0.0002993719583618481], "gradient_norm": 0.5628584623336792, "seconds": 0.3953038970000762, "nonpadding_tokens_per_second": 159568.37379720504, "ranks": [{"compute_seconds": 0.38979950700002064, "nonpadding_tokens": 9669, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3915356210000027, "nonpadding_tokens": 6547, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.38874886999997216, "nonpadding_tokens": 4232, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3927997690000211, "nonpadding_tokens": 5519, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3873777919999384, "nonpadding_tokens": 9295, "peak_allocated_bytes": 14980990464, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3906022410000105, "nonpadding_tokens": 11399, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3878252599999996, "nonpadding_tokens": 7747, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3878897649999544, "nonpadding_tokens": 8670, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004086120000010851, "prefetch": {"data_wait_seconds": 0.016770174001251803, "consumed_batches": 1781, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2331, "tokens": 173036080, "loss_tokens": 71699, "loss": 2.408187570607679, "lr": 0.0002993713699173665, "optimizer_lrs": [0.01995809132782443, 0.0002993713699173665], "gradient_norm": 0.41095682978630066, "seconds": 0.33528712700001506, "nonpadding_tokens_per_second": 213843.58129561227, "ranks": [{"compute_seconds": 0.33093718800000715, "nonpadding_tokens": 8833, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3324531339999339, "nonpadding_tokens": 2904, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33267928800000846, "nonpadding_tokens": 8062, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3300601880000613, "nonpadding_tokens": 1856, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3288733360000151, "nonpadding_tokens": 10719, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33093213600000126, "nonpadding_tokens": 11411, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3295176920000813, "nonpadding_tokens": 10836, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3290050800000017, "nonpadding_tokens": 17078, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035264999996797997, "prefetch": {"data_wait_seconds": 0.016780225001255644, "consumed_batches": 1782, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2332, "tokens": 173100294, "loss_tokens": 64214, "loss": 2.41952397452269, "lr": 0.00029937084267005567, "optimizer_lrs": [0.019958056178003714, 0.00029937084267005567], "gradient_norm": 0.5125706195831299, "seconds": 0.2551467949999733, "nonpadding_tokens_per_second": 251674.72709193436, "ranks": [{"compute_seconds": 0.1852534370000285, "nonpadding_tokens": 6132, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20669850299998416, "nonpadding_tokens": 2980, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20391932800009727, "nonpadding_tokens": 3464, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2077003770000374, "nonpadding_tokens": 27487, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2023806859999695, "nonpadding_tokens": 13723, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2066142219999847, "nonpadding_tokens": 4337, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2022696359999827, "nonpadding_tokens": 2987, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20855584299999919, "nonpadding_tokens": 3104, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.03035553900008381, "prefetch": {"data_wait_seconds": 0.01679065100127275, "consumed_batches": 1783, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2333, "tokens": 173175189, "loss_tokens": 74895, "loss": 2.351926029775018, "lr": 0.000299370227444793, "optimizer_lrs": [0.019958015162986203, 0.000299370227444793], "gradient_norm": 0.39620241522789, "seconds": 0.3322952600000235, "nonpadding_tokens_per_second": 225386.90440542155, "ranks": [{"compute_seconds": 0.32886290299995835, "nonpadding_tokens": 7843, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32762501399997745, "nonpadding_tokens": 4179, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32936221500006013, "nonpadding_tokens": 1803, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32539091700004974, "nonpadding_tokens": 13792, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32344394799997644, "nonpadding_tokens": 13115, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3265268279999418, "nonpadding_tokens": 14156, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3288408600000139, "nonpadding_tokens": 10598, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.32976155300002574, "nonpadding_tokens": 9409, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004425509999919086, "prefetch": {"data_wait_seconds": 0.016800104001234217, "consumed_batches": 1784, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2334, "tokens": 173308804, "loss_tokens": 133615, "loss": 2.382831736706208, "lr": 0.0002993691291189708, "optimizer_lrs": [0.019957941941264722, 0.0002993691291189708], "gradient_norm": 0.6060928106307983, "seconds": 0.34450189699998646, "nonpadding_tokens_per_second": 387849.82365425193, "ranks": [{"compute_seconds": 0.33825951300002544, "nonpadding_tokens": 6369, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3379915500000834, "nonpadding_tokens": 15604, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3410034099999848, "nonpadding_tokens": 2754, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3416991520000465, "nonpadding_tokens": 7079, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3417494999999917, "nonpadding_tokens": 28212, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33876371899998503, "nonpadding_tokens": 18965, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33614981099992747, "nonpadding_tokens": 25814, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34202895400005673, "nonpadding_tokens": 28818, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000449085000013838, "prefetch": {"data_wait_seconds": 0.01680793700127481, "consumed_batches": 1785, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2335, "tokens": 173416491, "loss_tokens": 107687, "loss": 2.1605921559705443, "lr": 0.00029936824322859004, "optimizer_lrs": [0.019957882881906003, 0.00029936824322859004], "gradient_norm": 0.3728090822696686, "seconds": 0.2959671959999923, "nonpadding_tokens_per_second": 363847.7556141147, "ranks": [{"compute_seconds": 0.29322347400000126, "nonpadding_tokens": 8160, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2933852980000893, "nonpadding_tokens": 15119, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.290315013000054, "nonpadding_tokens": 17356, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2932348900000079, "nonpadding_tokens": 23247, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2933555910000223, "nonpadding_tokens": 3740, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29172809499993946, "nonpadding_tokens": 13252, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29300591899993833, "nonpadding_tokens": 4444, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29020504599998276, "nonpadding_tokens": 22369, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004508479999003612, "prefetch": {"data_wait_seconds": 0.0168145250012941, "consumed_batches": 1786, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2336, "tokens": 173467758, "loss_tokens": 51267, "loss": 2.838143018413404, "lr": 0.0002993678212611752, "optimizer_lrs": [0.019957854750745015, 0.0002993678212611752], "gradient_norm": 0.43893736600875854, "seconds": 0.2766599490000772, "nonpadding_tokens_per_second": 185306.90902421044, "ranks": [{"compute_seconds": 0.27080591599997206, "nonpadding_tokens": 1384, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.271221970000056, "nonpadding_tokens": 9065, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2706914410000536, "nonpadding_tokens": 3415, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27335490200005097, "nonpadding_tokens": 10875, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2742730819999224, "nonpadding_tokens": 13390, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2722631699999738, "nonpadding_tokens": 4926, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2698301350000065, "nonpadding_tokens": 5003, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2703255070000523, "nonpadding_tokens": 3209, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039532800008146296, "prefetch": {"data_wait_seconds": 0.01682333500123434, "consumed_batches": 1787, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2337, "tokens": 173553904, "loss_tokens": 86146, "loss": 2.5774188949573977, "lr": 0.00029936711189587693, "optimizer_lrs": [0.01995780745972513, 0.00029936711189587693], "gradient_norm": 0.47520118951797485, "seconds": 0.3229556789999606, "nonpadding_tokens_per_second": 266742.4838812341, "ranks": [{"compute_seconds": 0.3200767910000195, "nonpadding_tokens": 10112, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31764013400004387, "nonpadding_tokens": 6927, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3202078629999505, "nonpadding_tokens": 10260, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3202914309999869, "nonpadding_tokens": 23188, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31308757900001183, "nonpadding_tokens": 16378, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31693971999993664, "nonpadding_tokens": 7907, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3127214720000211, "nonpadding_tokens": 7428, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3134033040000759, "nonpadding_tokens": 3946, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037271200005761784, "prefetch": {"data_wait_seconds": 0.01683121400124321, "consumed_batches": 1788, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2338, "tokens": 173637767, "loss_tokens": 83863, "loss": 2.2931954497215696, "lr": 0.0002993664209486435, "optimizer_lrs": [0.019957761396576235, 0.0002993664209486435], "gradient_norm": 0.3876557946205139, "seconds": 0.26077973799999654, "nonpadding_tokens_per_second": 321585.56735723506, "ranks": [{"compute_seconds": 0.25812605999999505, "nonpadding_tokens": 3652, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25772268299999723, "nonpadding_tokens": 13164, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2544112570000152, "nonpadding_tokens": 12227, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2573872399999573, "nonpadding_tokens": 13392, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25834304800002883, "nonpadding_tokens": 10961, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25676454200004173, "nonpadding_tokens": 4319, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2582716419999542, "nonpadding_tokens": 17707, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25429146399994806, "nonpadding_tokens": 8441, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003519519999599652, "prefetch": {"data_wait_seconds": 0.01684092500120471, "consumed_batches": 1789, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2339, "tokens": 173718886, "loss_tokens": 81119, "loss": 2.2303404180894737, "lr": 0.00029936575225140825, "optimizer_lrs": [0.019957716816760552, 0.00029936575225140825], "gradient_norm": 0.4559834599494934, "seconds": 0.24738392699998712, "nonpadding_tokens_per_second": 327907.3179237074, "ranks": [{"compute_seconds": 0.2424367109999821, "nonpadding_tokens": 11633, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24501247399996373, "nonpadding_tokens": 4586, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.241611789999979, "nonpadding_tokens": 13716, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24234160999992582, "nonpadding_tokens": 13714, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24463419899996097, "nonpadding_tokens": 13867, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2421890739999526, "nonpadding_tokens": 6528, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24062114200000906, "nonpadding_tokens": 4406, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2407826500000283, "nonpadding_tokens": 12669, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045113999999557564, "prefetch": {"data_wait_seconds": 0.016856526001177485, "consumed_batches": 1790, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2340, "tokens": 173783725, "loss_tokens": 64839, "loss": 1.661192651220716, "lr": 0.0002993652175038744, "optimizer_lrs": [0.01995768116692496, 0.0002993652175038744], "gradient_norm": 1.06270170211792, "seconds": 0.45334044699995957, "nonpadding_tokens_per_second": 143024.96154728896, "ranks": [{"compute_seconds": 0.45094901100003426, "nonpadding_tokens": 9999, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4484212590000425, "nonpadding_tokens": 4383, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4494156149999071, "nonpadding_tokens": 19605, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4502510759999723, "nonpadding_tokens": 11884, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.450979177000022, "nonpadding_tokens": 4175, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4477917090000574, "nonpadding_tokens": 3745, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4447322029999441, "nonpadding_tokens": 3634, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.44563191599991114, "nonpadding_tokens": 7414, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034047799999825656, "prefetch": {"data_wait_seconds": 0.01686466400117581, "consumed_batches": 1791, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2341, "tokens": 173835598, "loss_tokens": 51873, "loss": 2.324862524820234, "lr": 0.00029936478952913645, "optimizer_lrs": [0.019957652635275765, 0.00029936478952913645], "gradient_norm": 0.40637728571891785, "seconds": 0.33981165000000146, "nonpadding_tokens_per_second": 152652.21189444145, "ranks": [{"compute_seconds": 0.3342946430000211, "nonpadding_tokens": 14316, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33380476800005, "nonpadding_tokens": 4350, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33640297099998406, "nonpadding_tokens": 7691, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33685998999999356, "nonpadding_tokens": 1607, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33735885299995516, "nonpadding_tokens": 2549, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.334665780000023, "nonpadding_tokens": 8832, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3321208889999525, "nonpadding_tokens": 3655, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3327854679999973, "nonpadding_tokens": 8873, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004152830000521135, "prefetch": {"data_wait_seconds": 0.016875514001142733, "consumed_batches": 1792, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2342, "tokens": 173882702, "loss_tokens": 47104, "loss": 2.380565311597741, "lr": 0.00029936440077606694, "optimizer_lrs": [0.019957626718404464, 0.00029936440077606694], "gradient_norm": 0.4577471613883972, "seconds": 0.23582343399993988, "nonpadding_tokens_per_second": 199742.66001067564, "ranks": [{"compute_seconds": 0.23095578200002365, "nonpadding_tokens": 5160, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2320444579999048, "nonpadding_tokens": 9730, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22946651200004453, "nonpadding_tokens": 1546, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2299785590000738, "nonpadding_tokens": 9960, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2332708180000509, "nonpadding_tokens": 8953, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23090962100002344, "nonpadding_tokens": 3727, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2335259070000575, "nonpadding_tokens": 4326, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22890801400001237, "nonpadding_tokens": 3702, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037502800000766, "prefetch": {"data_wait_seconds": 0.016884262001170214, "consumed_batches": 1793, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2343, "tokens": 173926146, "loss_tokens": 43444, "loss": 2.3604811294425008, "lr": 0.00029936404212409657, "optimizer_lrs": [0.019957602808273106, 0.00029936404212409657], "gradient_norm": 0.5657017230987549, "seconds": 0.3342819000000645, "nonpadding_tokens_per_second": 129962.16666230392, "ranks": [{"compute_seconds": 0.3304077470000948, "nonpadding_tokens": 9918, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3289722060000031, "nonpadding_tokens": 3362, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33027880299994195, "nonpadding_tokens": 2389, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3267988200000218, "nonpadding_tokens": 4601, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32528805000004013, "nonpadding_tokens": 3404, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3284136899999339, "nonpadding_tokens": 3320, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33066020300009313, "nonpadding_tokens": 5116, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.33182569100006276, "nonpadding_tokens": 11334, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038633999997728097, "prefetch": {"data_wait_seconds": 0.016893322001195088, "consumed_batches": 1794, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2344, "tokens": 174023181, "loss_tokens": 97035, "loss": 2.442107873447725, "lr": 0.00029936324068714423, "optimizer_lrs": [0.01995754937914295, 0.00029936324068714423], "gradient_norm": 0.39760586619377136, "seconds": 0.29643221199989966, "nonpadding_tokens_per_second": 327342.96770700766, "ranks": [{"compute_seconds": 0.2933090300000458, "nonpadding_tokens": 3779, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28955958200003806, "nonpadding_tokens": 2068, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28904064599998947, "nonpadding_tokens": 30537, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2936479470000677, "nonpadding_tokens": 9693, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2877362389999689, "nonpadding_tokens": 9112, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2904456170000458, "nonpadding_tokens": 14927, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28765357500003574, "nonpadding_tokens": 17312, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28870020500005467, "nonpadding_tokens": 9607, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040411800000583753, "prefetch": {"data_wait_seconds": 0.01690443900118055, "consumed_batches": 1795, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2345, "tokens": 174085262, "loss_tokens": 62081, "loss": 2.279158478439458, "lr": 0.0002993627276801453, "optimizer_lrs": [0.019957515178676357, 0.0002993627276801453], "gradient_norm": 0.4102403223514557, "seconds": 0.26722953599994526, "nonpadding_tokens_per_second": 232313.39218436062, "ranks": [{"compute_seconds": 0.2595902319999368, "nonpadding_tokens": 3603, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2620259619999388, "nonpadding_tokens": 3854, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.264483549999909, "nonpadding_tokens": 14616, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.264489719999915, "nonpadding_tokens": 1566, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2648786839999957, "nonpadding_tokens": 6023, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2608230230000572, "nonpadding_tokens": 2963, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2647246420000329, "nonpadding_tokens": 19657, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2586611870000297, "nonpadding_tokens": 9799, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036445100010951137, "prefetch": {"data_wait_seconds": 0.016913676001152567, "consumed_batches": 1796, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2346, "tokens": 174137394, "loss_tokens": 52132, "loss": 2.2395322810365994, "lr": 0.0002993622967276251, "optimizer_lrs": [0.01995748644850834, 0.0002993622967276251], "gradient_norm": 0.40476834774017334, "seconds": 0.30926426499991067, "nonpadding_tokens_per_second": 168567.81044526777, "ranks": [{"compute_seconds": 0.3001432499999055, "nonpadding_tokens": 2757, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3013454990000355, "nonpadding_tokens": 5847, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30439528700003393, "nonpadding_tokens": 9015, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29924715200002083, "nonpadding_tokens": 3782, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30656486599991695, "nonpadding_tokens": 4203, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3004230450000023, "nonpadding_tokens": 15494, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.304802259999974, "nonpadding_tokens": 5976, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29806084500000907, "nonpadding_tokens": 5058, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00048622899998918, "prefetch": {"data_wait_seconds": 0.01692372800125952, "consumed_batches": 1797, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2347, "tokens": 174194485, "loss_tokens": 57091, "loss": 2.140764169264858, "lr": 0.00029936182461450016, "optimizer_lrs": [0.01995745497430001, 0.00029936182461450016], "gradient_norm": 0.43245434761047363, "seconds": 0.21791878900000938, "nonpadding_tokens_per_second": 261982.9169480083, "ranks": [{"compute_seconds": 0.18008364299998902, "nonpadding_tokens": 6127, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.1838110979999783, "nonpadding_tokens": 2151, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.1860720320000837, "nonpadding_tokens": 6220, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.1809733510000342, "nonpadding_tokens": 14524, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1798262900000509, "nonpadding_tokens": 13226, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.18561661799992635, "nonpadding_tokens": 6473, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.17955604600001607, "nonpadding_tokens": 2801, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.1878097459999708, "nonpadding_tokens": 5569, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005046100000072329, "prefetch": {"data_wait_seconds": 0.016934013001289827, "consumed_batches": 1798, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2348, "tokens": 174267871, "loss_tokens": 73386, "loss": 2.185544586160848, "lr": 0.00029936121749413294, "optimizer_lrs": [0.019957414499608864, 0.00029936121749413294], "gradient_norm": 0.42954573035240173, "seconds": 0.3590615349999098, "nonpadding_tokens_per_second": 204382.79472073898, "ranks": [{"compute_seconds": 0.3537258239999801, "nonpadding_tokens": 14798, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.354018801000052, "nonpadding_tokens": 2306, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35067538199996307, "nonpadding_tokens": 15740, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3554386289999911, "nonpadding_tokens": 12408, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34988245499994264, "nonpadding_tokens": 10413, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3526765599999635, "nonpadding_tokens": 4965, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3565503830000125, "nonpadding_tokens": 5961, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3500135300000693, "nonpadding_tokens": 6795, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004197649999468922, "prefetch": {"data_wait_seconds": 0.01694335800125657, "consumed_batches": 1799, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2349, "tokens": 174327750, "loss_tokens": 59879, "loss": 2.3007993202959303, "lr": 0.000299360721903465, "optimizer_lrs": [0.019957381460231002, 0.000299360721903465], "gradient_norm": 0.36250871419906616, "seconds": 0.3054374639999651, "nonpadding_tokens_per_second": 196043.40350339882, "ranks": [{"compute_seconds": 0.2983230589999266, "nonpadding_tokens": 15220, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29965071699996315, "nonpadding_tokens": 8101, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30222769900001367, "nonpadding_tokens": 4312, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2981687059999558, "nonpadding_tokens": 10730, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3030434020000712, "nonpadding_tokens": 3043, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3010591620000014, "nonpadding_tokens": 12489, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3027699990000201, "nonpadding_tokens": 2067, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2965988680000464, "nonpadding_tokens": 3917, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039448899997296394, "prefetch": {"data_wait_seconds": 0.016952384001299947, "consumed_batches": 1800, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2350, "tokens": 174410709, "loss_tokens": 82959, "loss": 2.6618188276739114, "lr": 0.0002993600349735543, "optimizer_lrs": [0.01995733566490362, 0.0002993600349735543], "gradient_norm": 0.3437305688858032, "seconds": 0.24568297000007533, "nonpadding_tokens_per_second": 337666.8720667719, "ranks": [{"compute_seconds": 0.2394996739999442, "nonpadding_tokens": 3360, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23964848900004654, "nonpadding_tokens": 13440, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24267436100001305, "nonpadding_tokens": 7029, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23749850499996228, "nonpadding_tokens": 11582, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24259659300003023, "nonpadding_tokens": 7782, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2399748279999585, "nonpadding_tokens": 4795, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2422972930000924, "nonpadding_tokens": 17521, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24328378199993494, "nonpadding_tokens": 17450, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033946099995318946, "prefetch": {"data_wait_seconds": 0.016960961001316832, "consumed_batches": 1801, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2351, "tokens": 174444279, "loss_tokens": 33570, "loss": 2.3586186513255885, "lr": 0.00029935975689747673, "optimizer_lrs": [0.01995731712649845, 0.00029935975689747673], "gradient_norm": 0.43141230940818787, "seconds": 0.2914747059999172, "nonpadding_tokens_per_second": 115172.94402900791, "ranks": [{"compute_seconds": 0.2892202489999818, "nonpadding_tokens": 3854, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2859785450000345, "nonpadding_tokens": 2021, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28302765199998703, "nonpadding_tokens": 4143, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28323275200000353, "nonpadding_tokens": 6471, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2822444070000074, "nonpadding_tokens": 3940, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2860961879999877, "nonpadding_tokens": 5942, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28228028099999847, "nonpadding_tokens": 5346, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28894528699993316, "nonpadding_tokens": 1853, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036197399992943247, "prefetch": {"data_wait_seconds": 0.01696933400133105, "consumed_batches": 1802, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2352, "tokens": 174500878, "loss_tokens": 56599, "loss": 2.1943502756232443, "lr": 0.00029935928792493424, "optimizer_lrs": [0.019957285861662285, 0.00029935928792493424], "gradient_norm": 0.3983197510242462, "seconds": 0.26111865600000783, "nonpadding_tokens_per_second": 216755.86442968788, "ranks": [{"compute_seconds": 0.25683724900000016, "nonpadding_tokens": 7720, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.257088600999964, "nonpadding_tokens": 3522, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25704886899995927, "nonpadding_tokens": 5278, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2569479919999367, "nonpadding_tokens": 8969, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25873594699999103, "nonpadding_tokens": 11280, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2560147589999815, "nonpadding_tokens": 11889, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25257545099998424, "nonpadding_tokens": 5149, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25316019700005654, "nonpadding_tokens": 2792, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003985209999655126, "prefetch": {"data_wait_seconds": 0.01697716700137164, "consumed_batches": 1803, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2353, "tokens": 174537906, "loss_tokens": 37028, "loss": 2.622524677271254, "lr": 0.0002993589810226711, "optimizer_lrs": [0.019957265401511406, 0.0002993589810226711], "gradient_norm": 0.501632034778595, "seconds": 0.24303443600001629, "nonpadding_tokens_per_second": 152357.01001646335, "ranks": [{"compute_seconds": 0.23943395300000248, "nonpadding_tokens": 7904, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23867980400007127, "nonpadding_tokens": 5293, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2386329220000789, "nonpadding_tokens": 6091, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2403182210000523, "nonpadding_tokens": 5658, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23474925799996527, "nonpadding_tokens": 1264, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23745666399997845, "nonpadding_tokens": 2397, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23431721899999047, "nonpadding_tokens": 4921, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23499498899991522, "nonpadding_tokens": 3500, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039621999997052626, "prefetch": {"data_wait_seconds": 0.01698765200133323, "consumed_batches": 1804, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2354, "tokens": 174587899, "loss_tokens": 49993, "loss": 2.534002416588322, "lr": 0.00029935856654520783, "optimizer_lrs": [0.019957237769680523, 0.00029935856654520783], "gradient_norm": 0.4567301273345947, "seconds": 0.30731294500003514, "nonpadding_tokens_per_second": 162677.8201614458, "ranks": [{"compute_seconds": 0.3031526859999758, "nonpadding_tokens": 9993, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30154190899997957, "nonpadding_tokens": 5061, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30450050299998566, "nonpadding_tokens": 2516, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29818596999996316, "nonpadding_tokens": 14478, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2959929269999293, "nonpadding_tokens": 5611, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2996141710000302, "nonpadding_tokens": 4412, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3028903880000371, "nonpadding_tokens": 5770, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.296437654999977, "nonpadding_tokens": 2152, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003891249999696811, "prefetch": {"data_wait_seconds": 0.01699744500133704, "consumed_batches": 1805, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2355, "tokens": 174650607, "loss_tokens": 62708, "loss": 2.159204666868661, "lr": 0.0002993580464624261, "optimizer_lrs": [0.019957203097495074, 0.0002993580464624261], "gradient_norm": 0.4558873474597931, "seconds": 0.2983242400000563, "nonpadding_tokens_per_second": 210200.82042273253, "ranks": [{"compute_seconds": 0.29100192100008826, "nonpadding_tokens": 6858, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29141001499999675, "nonpadding_tokens": 13034, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2933647129999599, "nonpadding_tokens": 3235, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2939985270000989, "nonpadding_tokens": 10191, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.295690501000081, "nonpadding_tokens": 9597, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.291778972999964, "nonpadding_tokens": 11994, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28743844299992816, "nonpadding_tokens": 5914, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28803044899996166, "nonpadding_tokens": 1885, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004705369999555842, "prefetch": {"data_wait_seconds": 0.017007299001264187, "consumed_batches": 1806, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2356, "tokens": 174693116, "loss_tokens": 42509, "loss": 2.262240238831777, "lr": 0.0002993576937849798, "optimizer_lrs": [0.019957179585665324, 0.0002993576937849798], "gradient_norm": 0.44647935032844543, "seconds": 0.2281878469999583, "nonpadding_tokens_per_second": 186289.50033437918, "ranks": [{"compute_seconds": 0.21989403400004903, "nonpadding_tokens": 2942, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22058268199998565, "nonpadding_tokens": 7620, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21784748499999296, "nonpadding_tokens": 5157, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21881691700002648, "nonpadding_tokens": 13289, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2240730880000683, "nonpadding_tokens": 2991, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2210020049999457, "nonpadding_tokens": 3858, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2256443549999858, "nonpadding_tokens": 1808, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.21697893500004284, "nonpadding_tokens": 4844, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003497720000495974, "prefetch": {"data_wait_seconds": 0.017015873001312798, "consumed_batches": 1807, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2357, "tokens": 174750922, "loss_tokens": 57806, "loss": 2.5283531878178738, "lr": 0.00029935721404039444, "optimizer_lrs": [0.019957147602692963, 0.00029935721404039444], "gradient_norm": 0.4634857177734375, "seconds": 0.3788713699999562, "nonpadding_tokens_per_second": 152574.2100808691, "ranks": [{"compute_seconds": 0.3765441519999513, "nonpadding_tokens": 11234, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3734807280000041, "nonpadding_tokens": 6951, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3718419829999675, "nonpadding_tokens": 3019, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.37258863800002473, "nonpadding_tokens": 11408, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3704286840001032, "nonpadding_tokens": 10560, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37381869000000734, "nonpadding_tokens": 3312, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37593737199995303, "nonpadding_tokens": 8045, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.37133106400006, "nonpadding_tokens": 3277, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003417800000988791, "prefetch": {"data_wait_seconds": 0.01702394100136928, "consumed_batches": 1808, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2358, "tokens": 174824383, "loss_tokens": 73461, "loss": 2.374599489865371, "lr": 0.0002993566041137533, "optimizer_lrs": [0.01995710694091689, 0.0002993566041137533], "gradient_norm": 0.38768577575683594, "seconds": 0.27368044499996813, "nonpadding_tokens_per_second": 268418.88539025345, "ranks": [{"compute_seconds": 0.27098472699992726, "nonpadding_tokens": 6556, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2681576469999527, "nonpadding_tokens": 13523, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2649111120000498, "nonpadding_tokens": 2777, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27061769800002367, "nonpadding_tokens": 14750, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2640507109999817, "nonpadding_tokens": 2747, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2669199549999348, "nonpadding_tokens": 3664, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2637378340000396, "nonpadding_tokens": 15970, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2645131649999257, "nonpadding_tokens": 13474, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038175400004547555, "prefetch": {"data_wait_seconds": 0.017033806001293783, "consumed_batches": 1809, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2359, "tokens": 174875673, "loss_tokens": 51290, "loss": 2.4768438352992788, "lr": 0.000299356178095817, "optimizer_lrs": [0.019957078539721132, 0.000299356178095817], "gradient_norm": 0.5207998752593994, "seconds": 0.25801429000000553, "nonpadding_tokens_per_second": 198787.43925384482, "ranks": [{"compute_seconds": 0.25305984200008425, "nonpadding_tokens": 2991, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25226155600000766, "nonpadding_tokens": 4209, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25533825500008334, "nonpadding_tokens": 6839, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2493111469999576, "nonpadding_tokens": 3602, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2483249160000014, "nonpadding_tokens": 4287, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2507520700000896, "nonpadding_tokens": 9305, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25453410800002985, "nonpadding_tokens": 2832, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24837567500003388, "nonpadding_tokens": 17225, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00048472899993612373, "prefetch": {"data_wait_seconds": 0.017045172001260767, "consumed_batches": 1810, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2360, "tokens": 174925707, "loss_tokens": 50034, "loss": 2.329133689091418, "lr": 0.0002993557623747616, "optimizer_lrs": [0.019957050824984106, 0.0002993557623747616], "gradient_norm": 0.4578472077846527, "seconds": 0.24403840899992701, "nonpadding_tokens_per_second": 205025.10324108432, "ranks": [{"compute_seconds": 0.2387866930000655, "nonpadding_tokens": 6695, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23822567499996694, "nonpadding_tokens": 7652, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2349573369999689, "nonpadding_tokens": 5576, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2353454880000072, "nonpadding_tokens": 4776, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23389284500001395, "nonpadding_tokens": 1556, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23657955100009076, "nonpadding_tokens": 5088, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24137224400010382, "nonpadding_tokens": 9689, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23396884200008117, "nonpadding_tokens": 9002, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004024430000981738, "prefetch": {"data_wait_seconds": 0.017053776001333176, "consumed_batches": 1811, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2361, "tokens": 175018143, "loss_tokens": 92436, "loss": 2.117385271971959, "lr": 0.00029935499399312, "optimizer_lrs": [0.019956999599541336, 0.00029935499399312], "gradient_norm": 0.6211256980895996, "seconds": 0.35291385499999706, "nonpadding_tokens_per_second": 261922.2756216266, "ranks": [{"compute_seconds": 0.35061658300003273, "nonpadding_tokens": 15384, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34661268999991535, "nonpadding_tokens": 4663, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34511049799994, "nonpadding_tokens": 19096, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3497020849999899, "nonpadding_tokens": 16217, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3504333340000585, "nonpadding_tokens": 11441, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34723447499993654, "nonpadding_tokens": 21043, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3496966650000104, "nonpadding_tokens": 2033, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.34552848700002414, "nonpadding_tokens": 2559, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039595500004452333, "prefetch": {"data_wait_seconds": 0.01706349500132376, "consumed_batches": 1812, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2362, "tokens": 175084604, "loss_tokens": 66461, "loss": 2.1139942503874454, "lr": 0.0002993544412484191, "optimizer_lrs": [0.019956962749894606, 0.0002993544412484191], "gradient_norm": 0.39696258306503296, "seconds": 0.2946984949999205, "nonpadding_tokens_per_second": 225522.0203958555, "ranks": [{"compute_seconds": 0.28900210800009063, "nonpadding_tokens": 12524, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29111600900000667, "nonpadding_tokens": 9678, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2920977699999412, "nonpadding_tokens": 11112, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.289081527999997, "nonpadding_tokens": 5027, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2922689069999933, "nonpadding_tokens": 9497, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29014179900002546, "nonpadding_tokens": 8559, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2920270829999936, "nonpadding_tokens": 5987, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28824075099998936, "nonpadding_tokens": 4077, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003911709999329105, "prefetch": {"data_wait_seconds": 0.017072446001364483, "consumed_batches": 1813, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2363, "tokens": 175135212, "loss_tokens": 50608, "loss": 2.126397074079197, "lr": 0.0002993540201920026, "optimizer_lrs": [0.01995693467946684, 0.0002993540201920026], "gradient_norm": 0.3856450617313385, "seconds": 0.29551984599993375, "nonpadding_tokens_per_second": 171250.76601458213, "ranks": [{"compute_seconds": 0.291142258999912, "nonpadding_tokens": 10335, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28929944000003616, "nonpadding_tokens": 2496, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28609678500004065, "nonpadding_tokens": 3863, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2867204589999801, "nonpadding_tokens": 15384, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29282932399996753, "nonpadding_tokens": 2371, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2889912840000761, "nonpadding_tokens": 2192, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28533889999994244, "nonpadding_tokens": 7103, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29185929999994187, "nonpadding_tokens": 6864, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003917349999937869, "prefetch": {"data_wait_seconds": 0.017080413001352213, "consumed_batches": 1814, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2364, "tokens": 175183282, "loss_tokens": 48070, "loss": 2.566108507125026, "lr": 0.00029935362012482777, "optimizer_lrs": [0.019956908008321853, 0.00029935362012482777], "gradient_norm": 0.45050492882728577, "seconds": 0.24324993499999437, "nonpadding_tokens_per_second": 197615.6745941211, "ranks": [{"compute_seconds": 0.23843109099993853, "nonpadding_tokens": 1811, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23697097399997347, "nonpadding_tokens": 3617, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23397310999996535, "nonpadding_tokens": 4680, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23969404100000702, "nonpadding_tokens": 5760, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24078615399992032, "nonpadding_tokens": 12769, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23744828799999596, "nonpadding_tokens": 2214, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2332675710000558, "nonpadding_tokens": 4518, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23393463000002157, "nonpadding_tokens": 12701, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038115999996080063, "prefetch": {"data_wait_seconds": 0.017089531001332148, "consumed_batches": 1815, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2365, "tokens": 175243412, "loss_tokens": 60130, "loss": 2.7655434059537667, "lr": 0.0002993531195132173, "optimizer_lrs": [0.019956874634214486, 0.0002993531195132173], "gradient_norm": 0.5909397006034851, "seconds": 0.3206936210000322, "nonpadding_tokens_per_second": 187499.8318098568, "ranks": [{"compute_seconds": 0.31811815700007173, "nonpadding_tokens": 10072, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3133794300000545, "nonpadding_tokens": 7542, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3102606840000135, "nonpadding_tokens": 6405, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31095186999993985, "nonpadding_tokens": 13791, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3163873139999396, "nonpadding_tokens": 1347, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31345261700005267, "nonpadding_tokens": 2505, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31607093600007374, "nonpadding_tokens": 13125, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3164802169999348, "nonpadding_tokens": 5343, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038743199991131405, "prefetch": {"data_wait_seconds": 0.017098278001370204, "consumed_batches": 1816, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2366, "tokens": 175309158, "loss_tokens": 65746, "loss": 2.3350313136920877, "lr": 0.00029935257192442076, "optimizer_lrs": [0.01995683812829472, 0.00029935257192442076], "gradient_norm": 0.42126432061195374, "seconds": 0.27763126500008184, "nonpadding_tokens_per_second": 236810.5047534204, "ranks": [{"compute_seconds": 0.267591234000065, "nonpadding_tokens": 4033, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2692379729999175, "nonpadding_tokens": 3834, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27205877899996267, "nonpadding_tokens": 10322, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2666617430000997, "nonpadding_tokens": 15919, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27331194399994274, "nonpadding_tokens": 10312, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.268458881000015, "nonpadding_tokens": 10489, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2748572539999259, "nonpadding_tokens": 8490, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27476960799992867, "nonpadding_tokens": 2347, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004527670000697981, "prefetch": {"data_wait_seconds": 0.017109618001384774, "consumed_batches": 1817, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2367, "tokens": 175360754, "loss_tokens": 51596, "loss": 2.196184660632607, "lr": 0.00029935214202704476, "optimizer_lrs": [0.01995680946846965, 0.00029935214202704476], "gradient_norm": 0.45300447940826416, "seconds": 0.25531585099997756, "nonpadding_tokens_per_second": 202086.94367356194, "ranks": [{"compute_seconds": 0.1881823160000522, "nonpadding_tokens": 3436, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20254610599999978, "nonpadding_tokens": 8330, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.19933355900002425, "nonpadding_tokens": 2461, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20370352200006892, "nonpadding_tokens": 3842, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1984621070000685, "nonpadding_tokens": 12937, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2012600499999735, "nonpadding_tokens": 11546, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20441947699998764, "nonpadding_tokens": 2825, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.20449081899994326, "nonpadding_tokens": 6219, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.03369767500009857, "prefetch": {"data_wait_seconds": 0.01712060400139137, "consumed_batches": 1818, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2368, "tokens": 175401672, "loss_tokens": 40918, "loss": 2.4956108864069604, "lr": 0.0002993518009974629, "optimizer_lrs": [0.019956786733164197, 0.0002993518009974629], "gradient_norm": 0.5550371408462524, "seconds": 0.3203191479999532, "nonpadding_tokens_per_second": 127741.34876259716, "ranks": [{"compute_seconds": 0.31280973999992057, "nonpadding_tokens": 5443, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31180776500002594, "nonpadding_tokens": 3403, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30924074400002155, "nonpadding_tokens": 3954, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3175947539999697, "nonpadding_tokens": 3329, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3167033359999323, "nonpadding_tokens": 3893, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3126668099999961, "nonpadding_tokens": 8984, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3090529119999701, "nonpadding_tokens": 6233, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3165771940000468, "nonpadding_tokens": 5679, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039315699996222975, "prefetch": {"data_wait_seconds": 0.017130870001395238, "consumed_batches": 1819, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2369, "tokens": 175473052, "loss_tokens": 71380, "loss": 2.1926404017231715, "lr": 0.00029935120586916004, "optimizer_lrs": [0.019956747057944004, 0.00029935120586916004], "gradient_norm": 0.6912471055984497, "seconds": 0.3291117580000673, "nonpadding_tokens_per_second": 216886.81204755194, "ranks": [{"compute_seconds": 0.25940788100001555, "nonpadding_tokens": 4847, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28265621700006704, "nonpadding_tokens": 11870, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28545707399996445, "nonpadding_tokens": 8656, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2882550589999937, "nonpadding_tokens": 8355, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.286527971000055, "nonpadding_tokens": 4931, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28154015900008744, "nonpadding_tokens": 11214, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2863642439999694, "nonpadding_tokens": 8966, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2791614440000103, "nonpadding_tokens": 12541, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.02726458500001172, "prefetch": {"data_wait_seconds": 0.01714012700138312, "consumed_batches": 1820, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2370, "tokens": 175536561, "loss_tokens": 63509, "loss": 2.3851422633012644, "lr": 0.0002993506761360346, "optimizer_lrs": [0.01995671174240231, 0.0002993506761360346], "gradient_norm": 0.42915549874305725, "seconds": 0.2816192120000096, "nonpadding_tokens_per_second": 225513.73377182035, "ranks": [{"compute_seconds": 0.2749925409999605, "nonpadding_tokens": 7469, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2752441769999905, "nonpadding_tokens": 10154, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27225954900006855, "nonpadding_tokens": 1650, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27274844400005804, "nonpadding_tokens": 2831, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.278889147999962, "nonpadding_tokens": 12485, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27484774300000936, "nonpadding_tokens": 11204, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27851332799991724, "nonpadding_tokens": 6484, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2721292029999631, "nonpadding_tokens": 11232, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039677699999174365, "prefetch": {"data_wait_seconds": 0.017149310001357776, "consumed_batches": 1821, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2371, "tokens": 175574773, "loss_tokens": 38212, "loss": 2.3259484095310374, "lr": 0.0002993503573030901, "optimizer_lrs": [0.019956690486872673, 0.0002993503573030901], "gradient_norm": 0.4872663617134094, "seconds": 0.2776306250000289, "nonpadding_tokens_per_second": 137636.11273070477, "ranks": [{"compute_seconds": 0.27507438599991474, "nonpadding_tokens": 7265, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2711494520000315, "nonpadding_tokens": 6987, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.269739598000001, "nonpadding_tokens": 3266, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27052238999999645, "nonpadding_tokens": 4286, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2692024790000005, "nonpadding_tokens": 3176, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2717345440000827, "nonpadding_tokens": 4445, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27462977100003627, "nonpadding_tokens": 3438, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26988942599996335, "nonpadding_tokens": 5349, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004175390000682455, "prefetch": {"data_wait_seconds": 0.017159047001385375, "consumed_batches": 1822, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2372, "tokens": 175663306, "loss_tokens": 88533, "loss": 2.3374274987857633, "lr": 0.0002993496183022596, "optimizer_lrs": [0.01995664122015064, 0.0002993496183022596], "gradient_norm": 0.42512694001197815, "seconds": 0.3569583319999765, "nonpadding_tokens_per_second": 248020.54487414466, "ranks": [{"compute_seconds": 0.3539677330001041, "nonpadding_tokens": 15433, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35070927999993273, "nonpadding_tokens": 7687, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35328728899992257, "nonpadding_tokens": 26161, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35365712099996927, "nonpadding_tokens": 4930, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35402424799997334, "nonpadding_tokens": 4009, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3514823579999984, "nonpadding_tokens": 5547, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35352297299994007, "nonpadding_tokens": 6583, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3544944430000214, "nonpadding_tokens": 18183, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003536660000236225, "prefetch": {"data_wait_seconds": 0.017167667001402265, "consumed_batches": 1823, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2373, "tokens": 175749759, "loss_tokens": 86453, "loss": 2.2625328213017477, "lr": 0.00029934889625914384, "optimizer_lrs": [0.019956593083942924, 0.00029934889625914384], "gradient_norm": 0.41953516006469727, "seconds": 0.35771842100007234, "nonpadding_tokens_per_second": 241678.91538351198, "ranks": [{"compute_seconds": 0.3422844949999444, "nonpadding_tokens": 22798, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3496600180000087, "nonpadding_tokens": 3257, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3518893850000495, "nonpadding_tokens": 13682, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34772048299998914, "nonpadding_tokens": 9571, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3456676170000037, "nonpadding_tokens": 10317, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3504052399999864, "nonpadding_tokens": 3558, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3536334350000061, "nonpadding_tokens": 14094, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.35321016299997154, "nonpadding_tokens": 9176, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000462378999941393, "prefetch": {"data_wait_seconds": 0.01717886900139831, "consumed_batches": 1824, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2374, "tokens": 175828310, "loss_tokens": 78551, "loss": 2.3105026909269135, "lr": 0.0002993482398659136, "optimizer_lrs": [0.01995654932439424, 0.0002993482398659136], "gradient_norm": 0.41508930921554565, "seconds": 0.23300793999999314, "nonpadding_tokens_per_second": 337117.2673343334, "ranks": [{"compute_seconds": 0.23010236000004625, "nonpadding_tokens": 1574, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22690948499996466, "nonpadding_tokens": 5923, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22901103700007752, "nonpadding_tokens": 18475, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23055487299996003, "nonpadding_tokens": 7914, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23000365600000805, "nonpadding_tokens": 7303, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.225716637000005, "nonpadding_tokens": 13466, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22978939400002218, "nonpadding_tokens": 11205, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.22382026200000382, "nonpadding_tokens": 12691, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003588179999951535, "prefetch": {"data_wait_seconds": 0.017187928001320074, "consumed_batches": 1825, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2375, "tokens": 175895864, "loss_tokens": 67554, "loss": 2.391305242842763, "lr": 0.0002993476751027144, "optimizer_lrs": [0.019956511673514293, 0.0002993476751027144], "gradient_norm": 0.3897915184497833, "seconds": 0.3084868150000375, "nonpadding_tokens_per_second": 218985.04803192898, "ranks": [{"compute_seconds": 0.30565893899995444, "nonpadding_tokens": 4639, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.302288112000042, "nonpadding_tokens": 10596, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2991140550000182, "nonpadding_tokens": 19168, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30008506299998317, "nonpadding_tokens": 1960, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29825609199997416, "nonpadding_tokens": 4676, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3011171519999607, "nonpadding_tokens": 5257, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3050839850000102, "nonpadding_tokens": 14174, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.305988794999962, "nonpadding_tokens": 7084, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041241700000682613, "prefetch": {"data_wait_seconds": 0.01719893900128966, "consumed_batches": 1826, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2376, "tokens": 175950418, "loss_tokens": 54554, "loss": 2.3158667902445282, "lr": 0.00029934721884369934, "optimizer_lrs": [0.019956481256246623, 0.00029934721884369934], "gradient_norm": 0.45837321877479553, "seconds": 0.3090892860000167, "nonpadding_tokens_per_second": 176499.1621223553, "ranks": [{"compute_seconds": 0.30581918900008986, "nonpadding_tokens": 6592, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30346804599992083, "nonpadding_tokens": 6322, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3008930450000662, "nonpadding_tokens": 5912, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3010018909999417, "nonpadding_tokens": 11757, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3065795129999742, "nonpadding_tokens": 4314, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30383199700008845, "nonpadding_tokens": 3429, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29963650900003813, "nonpadding_tokens": 11590, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2999082889999727, "nonpadding_tokens": 4638, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042430400003468094, "prefetch": {"data_wait_seconds": 0.017208792001341067, "consumed_batches": 1827, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2377, "tokens": 176013453, "loss_tokens": 63035, "loss": 2.397353156976283, "lr": 0.0002993466914562164, "optimizer_lrs": [0.019956446097081096, 0.0002993466914562164], "gradient_norm": 0.7264533042907715, "seconds": 0.30004046700003073, "nonpadding_tokens_per_second": 210088.32785210118, "ranks": [{"compute_seconds": 0.2971778370000493, "nonpadding_tokens": 3905, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29367088800006513, "nonpadding_tokens": 14768, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2910064579999698, "nonpadding_tokens": 21427, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2967721300000221, "nonpadding_tokens": 5167, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29740609600003154, "nonpadding_tokens": 3835, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29464200500001425, "nonpadding_tokens": 3581, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29012804399997094, "nonpadding_tokens": 4717, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2908047409999881, "nonpadding_tokens": 5635, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044167600003675034, "prefetch": {"data_wait_seconds": 0.017217985001323655, "consumed_batches": 1828, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2378, "tokens": 176069164, "loss_tokens": 55711, "loss": 2.6276778822853655, "lr": 0.00029934622516873146, "optimizer_lrs": [0.019956415011248764, 0.00029934622516873146], "gradient_norm": 0.43971115350723267, "seconds": 0.2358587639999996, "nonpadding_tokens_per_second": 236204.91795674842, "ranks": [{"compute_seconds": 0.2288915569999972, "nonpadding_tokens": 3960, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22904263999998875, "nonpadding_tokens": 5177, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22574032699992586, "nonpadding_tokens": 10876, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23230124600002, "nonpadding_tokens": 11396, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22455823799998598, "nonpadding_tokens": 8395, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23009082900000521, "nonpadding_tokens": 3634, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23325953400001254, "nonpadding_tokens": 4698, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2329259880000336, "nonpadding_tokens": 7575, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005199919999085978, "prefetch": {"data_wait_seconds": 0.017228254001295795, "consumed_batches": 1829, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2379, "tokens": 176135062, "loss_tokens": 65898, "loss": 2.1672333568545326, "lr": 0.00029934567340431345, "optimizer_lrs": [0.01995637822695423, 0.00029934567340431345], "gradient_norm": 0.4277808666229248, "seconds": 0.4898018989999855, "nonpadding_tokens_per_second": 134540.10720363082, "ranks": [{"compute_seconds": 0.48623066900006506, "nonpadding_tokens": 16637, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.48253021099992566, "nonpadding_tokens": 4759, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4854955810000092, "nonpadding_tokens": 10327, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4802183360000072, "nonpadding_tokens": 5416, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4864468299999771, "nonpadding_tokens": 12741, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.48530684499996823, "nonpadding_tokens": 9672, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4865027910000208, "nonpadding_tokens": 2503, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.4870595190000131, "nonpadding_tokens": 3843, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00048261200004162674, "prefetch": {"data_wait_seconds": 0.01723829200132343, "consumed_batches": 1830, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2380, "tokens": 176195545, "loss_tokens": 60483, "loss": 2.1573058235371922, "lr": 0.00029934516677539017, "optimizer_lrs": [0.01995634445169268, 0.00029934516677539017], "gradient_norm": 0.4161628186702728, "seconds": 0.2948979989999998, "nonpadding_tokens_per_second": 205098.03459195409, "ranks": [{"compute_seconds": 0.21989577599993027, "nonpadding_tokens": 10547, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2250961820000157, "nonpadding_tokens": 5903, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2277905669999427, "nonpadding_tokens": 2426, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2283675910000511, "nonpadding_tokens": 2828, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22156635299995742, "nonpadding_tokens": 12867, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2242323860000397, "nonpadding_tokens": 10085, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22940370200001325, "nonpadding_tokens": 11377, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23085538099996938, "nonpadding_tokens": 4450, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.034512729999960357, "prefetch": {"data_wait_seconds": 0.0172479290012717, "consumed_batches": 1831, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2381, "tokens": 176263796, "loss_tokens": 68251, "loss": 2.2096445931195148, "lr": 0.0002993445948438303, "optimizer_lrs": [0.01995630632292202, 0.0002993445948438303], "gradient_norm": 0.3486596345901489, "seconds": 0.3057076640000105, "nonpadding_tokens_per_second": 223255.77025768533, "ranks": [{"compute_seconds": 0.30187615099998766, "nonpadding_tokens": 4446, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29761986800008344, "nonpadding_tokens": 4021, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29569400699995185, "nonpadding_tokens": 8738, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29618785300010586, "nonpadding_tokens": 16821, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29482374700000946, "nonpadding_tokens": 9356, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29713967899999716, "nonpadding_tokens": 16478, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30021323800008304, "nonpadding_tokens": 6523, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3031606940000984, "nonpadding_tokens": 1868, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003819859999794062, "prefetch": {"data_wait_seconds": 0.01725736900125696, "consumed_batches": 1832, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2382, "tokens": 176314946, "loss_tokens": 51150, "loss": 2.146503696236559, "lr": 0.0002993441660524268, "optimizer_lrs": [0.019956277736828454, 0.0002993441660524268], "gradient_norm": 0.4436103403568268, "seconds": 0.2769237920000478, "nonpadding_tokens_per_second": 184707.8563765701, "ranks": [{"compute_seconds": 0.2715029979999599, "nonpadding_tokens": 5161, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.270687819999921, "nonpadding_tokens": 3828, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2677508269999862, "nonpadding_tokens": 4961, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2744111399999838, "nonpadding_tokens": 2148, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27373290900004577, "nonpadding_tokens": 15205, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26942106100000274, "nonpadding_tokens": 8926, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26674129499997434, "nonpadding_tokens": 7566, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2742884570000115, "nonpadding_tokens": 3355, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004350319999275598, "prefetch": {"data_wait_seconds": 0.01726720800122905, "consumed_batches": 1833, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2383, "tokens": 176374464, "loss_tokens": 59518, "loss": 2.221543797674653, "lr": 0.00029934366693584724, "optimizer_lrs": [0.01995624446238982, 0.00029934366693584724], "gradient_norm": 0.5180232524871826, "seconds": 0.26769005000005563, "nonpadding_tokens_per_second": 222339.23151042644, "ranks": [{"compute_seconds": 0.26376298799993947, "nonpadding_tokens": 11010, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2616657349999514, "nonpadding_tokens": 14426, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2632883789999596, "nonpadding_tokens": 4808, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2587808989999303, "nonpadding_tokens": 4139, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2650727259999712, "nonpadding_tokens": 5207, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.260436587000072, "nonpadding_tokens": 4510, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26463493000005656, "nonpadding_tokens": 2704, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.26488627899993844, "nonpadding_tokens": 12714, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003935659999569907, "prefetch": {"data_wait_seconds": 0.017276855001185254, "consumed_batches": 1834, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2384, "tokens": 176452127, "loss_tokens": 77663, "loss": 2.472735166681689, "lr": 0.00029934301537091736, "optimizer_lrs": [0.019956201024727824, 0.00029934301537091736], "gradient_norm": 0.403655081987381, "seconds": 0.38863436300005105, "nonpadding_tokens_per_second": 199835.64860421207, "ranks": [{"compute_seconds": 0.3816232860000355, "nonpadding_tokens": 19134, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.38194080900007066, "nonpadding_tokens": 7608, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.38456827400000293, "nonpadding_tokens": 3146, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.379572863999897, "nonpadding_tokens": 2901, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3852547600000662, "nonpadding_tokens": 4266, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3823318349999454, "nonpadding_tokens": 8019, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3850061470000128, "nonpadding_tokens": 18514, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.38592506999998477, "nonpadding_tokens": 14075, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004579179999382177, "prefetch": {"data_wait_seconds": 0.01728666800120493, "consumed_batches": 1835, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2385, "tokens": 176526847, "loss_tokens": 74720, "loss": 2.271194124732334, "lr": 0.0002993423881923418, "optimizer_lrs": [0.019956159212822788, 0.0002993423881923418], "gradient_norm": 0.4088677763938904, "seconds": 0.24718085899996822, "nonpadding_tokens_per_second": 302288.7787602098, "ranks": [{"compute_seconds": 0.23972935100005088, "nonpadding_tokens": 9503, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24098054599994612, "nonpadding_tokens": 1841, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24430434800001422, "nonpadding_tokens": 10586, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24428457900000922, "nonpadding_tokens": 3799, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24466953399996783, "nonpadding_tokens": 8326, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24187563400005274, "nonpadding_tokens": 18218, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2380020919999879, "nonpadding_tokens": 6643, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.23833750100004636, "nonpadding_tokens": 15804, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003516980000313197, "prefetch": {"data_wait_seconds": 0.017295149001213872, "consumed_batches": 1836, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2386, "tokens": 176610158, "loss_tokens": 83311, "loss": 2.510189043163568, "lr": 0.0002993416885513985, "optimizer_lrs": [0.019956112570093235, 0.0002993416885513985], "gradient_norm": 0.4539577066898346, "seconds": 0.2733705010000449, "nonpadding_tokens_per_second": 304754.90111490234, "ranks": [{"compute_seconds": 0.26986935000002177, "nonpadding_tokens": 9567, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26772687200002565, "nonpadding_tokens": 18479, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2649117280000155, "nonpadding_tokens": 10982, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26996574400004647, "nonpadding_tokens": 9231, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26481452800010175, "nonpadding_tokens": 5952, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2676099159999694, "nonpadding_tokens": 10060, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26355782699999963, "nonpadding_tokens": 10166, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27092752399994424, "nonpadding_tokens": 8874, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003778709999551211, "prefetch": {"data_wait_seconds": 0.017304236001180584, "consumed_batches": 1837, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2387, "tokens": 176656306, "loss_tokens": 46148, "loss": 2.097990263662564, "lr": 0.0002993413008434718, "optimizer_lrs": [0.01995608672289812, 0.0002993413008434718], "gradient_norm": 0.4383176565170288, "seconds": 0.29259292700010064, "nonpadding_tokens_per_second": 157720.83239723742, "ranks": [{"compute_seconds": 0.2328794709999329, "nonpadding_tokens": 4191, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2749453870000025, "nonpadding_tokens": 7746, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2720251250000274, "nonpadding_tokens": 8136, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27252366400000483, "nonpadding_tokens": 2184, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27821825599994554, "nonpadding_tokens": 2888, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.276120461000005, "nonpadding_tokens": 4745, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27068082900007084, "nonpadding_tokens": 2301, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2787548410000227, "nonpadding_tokens": 13957, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.011681478000014067, "prefetch": {"data_wait_seconds": 0.017330554001162568, "consumed_batches": 1838, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2388, "tokens": 176726745, "loss_tokens": 70439, "loss": 2.003937582518207, "lr": 0.00029934070883754974, "optimizer_lrs": [0.01995604725583665, 0.00029934070883754974], "gradient_norm": 0.7839570045471191, "seconds": 0.307003846999919, "nonpadding_tokens_per_second": 229440.12164127242, "ranks": [{"compute_seconds": 0.27149381900005665, "nonpadding_tokens": 4986, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2876511219999429, "nonpadding_tokens": 1539, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28542736599990803, "nonpadding_tokens": 3163, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28576331599992955, "nonpadding_tokens": 5754, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29190748000007716, "nonpadding_tokens": 9555, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2884773900000255, "nonpadding_tokens": 11766, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2911650370000416, "nonpadding_tokens": 18271, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2850559500000145, "nonpadding_tokens": 15405, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.011095597000007729, "prefetch": {"data_wait_seconds": 0.01734116300117705, "consumed_batches": 1839, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2389, "tokens": 176764181, "loss_tokens": 37436, "loss": 2.214331625974997, "lr": 0.0002993403940980205, "optimizer_lrs": [0.019956026273201366, 0.0002993403940980205], "gradient_norm": 0.4430238902568817, "seconds": 0.26630946199998107, "nonpadding_tokens_per_second": 140573.30039592308, "ranks": [{"compute_seconds": 0.26322727699994175, "nonpadding_tokens": 7228, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2613302830000066, "nonpadding_tokens": 3178, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2628391590000092, "nonpadding_tokens": 4614, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2635784630000444, "nonpadding_tokens": 2512, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26387541600001896, "nonpadding_tokens": 2222, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2605383159999519, "nonpadding_tokens": 3488, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25724569900000915, "nonpadding_tokens": 10047, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.25775393499998245, "nonpadding_tokens": 4147, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000370034999946256, "prefetch": {"data_wait_seconds": 0.017349040001207072, "consumed_batches": 1840, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2390, "tokens": 176827744, "loss_tokens": 63563, "loss": 2.2858441133206426, "lr": 0.0002993398595266711, "optimizer_lrs": [0.019955990635111408, 0.0002993398595266711], "gradient_norm": 0.5595579743385315, "seconds": 0.3270275660000834, "nonpadding_tokens_per_second": 194365.87801281494, "ranks": [{"compute_seconds": 0.3243905949999544, "nonpadding_tokens": 10843, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32015270299996246, "nonpadding_tokens": 5186, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3227818689999822, "nonpadding_tokens": 6435, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32399598799997875, "nonpadding_tokens": 13090, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.324483999999984, "nonpadding_tokens": 4362, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3189522089999173, "nonpadding_tokens": 18755, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.316276380999966, "nonpadding_tokens": 2341, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3242830579999918, "nonpadding_tokens": 2551, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000400836000039817, "prefetch": {"data_wait_seconds": 0.017359197001113102, "consumed_batches": 1841, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2391, "tokens": 176906565, "loss_tokens": 78821, "loss": 2.620572642443004, "lr": 0.0002993391963339784, "optimizer_lrs": [0.01995594642226523, 0.0002993391963339784], "gradient_norm": 0.4368712604045868, "seconds": 0.32283627599997544, "nonpadding_tokens_per_second": 244151.62068096088, "ranks": [{"compute_seconds": 0.3201193700000431, "nonpadding_tokens": 15519, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3162631689999671, "nonpadding_tokens": 25124, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31395765599995684, "nonpadding_tokens": 9402, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3145671500000162, "nonpadding_tokens": 5159, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31989341700000296, "nonpadding_tokens": 8175, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3176611960000173, "nonpadding_tokens": 7249, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31264585399992484, "nonpadding_tokens": 4251, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3133919220000507, "nonpadding_tokens": 3942, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004157370000257288, "prefetch": {"data_wait_seconds": 0.017368720001016413, "consumed_batches": 1842, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2392, "tokens": 176965361, "loss_tokens": 58796, "loss": 2.574025550207497, "lr": 0.0002993387014135174, "optimizer_lrs": [0.019955913427567827, 0.0002993387014135174], "gradient_norm": 0.4528815746307373, "seconds": 0.2825329209999836, "nonpadding_tokens_per_second": 208103.1824252559, "ranks": [{"compute_seconds": 0.2762948700000152, "nonpadding_tokens": 4074, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.276718849999952, "nonpadding_tokens": 4832, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2788300810000237, "nonpadding_tokens": 6434, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27937113699999827, "nonpadding_tokens": 18180, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27968273900000895, "nonpadding_tokens": 11501, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2772860249999667, "nonpadding_tokens": 3644, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.272942053999941, "nonpadding_tokens": 3091, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.28001521399994544, "nonpadding_tokens": 7040, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044256299997869064, "prefetch": {"data_wait_seconds": 0.017377675001014836, "consumed_batches": 1843, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2393, "tokens": 177017365, "loss_tokens": 52004, "loss": 2.268190307476348, "lr": 0.000299338263511285, "optimizer_lrs": [0.019955884234085668, 0.000299338263511285], "gradient_norm": 0.40793266892433167, "seconds": 0.39563920699993105, "nonpadding_tokens_per_second": 131442.98916767637, "ranks": [{"compute_seconds": 0.39187630699996134, "nonpadding_tokens": 3370, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.38812589500003014, "nonpadding_tokens": 8288, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3897988979999809, "nonpadding_tokens": 7109, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.39298181999993176, "nonpadding_tokens": 3162, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.39098294999996597, "nonpadding_tokens": 2332, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3883528019999858, "nonpadding_tokens": 4614, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.38385511800004224, "nonpadding_tokens": 13018, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3843848470000921, "nonpadding_tokens": 10111, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004282009999769798, "prefetch": {"data_wait_seconds": 0.01738699900097629, "consumed_batches": 1844, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2394, "tokens": 177066431, "loss_tokens": 49066, "loss": 2.404272434068398, "lr": 0.0002993378502160653, "optimizer_lrs": [0.01995585668107102, 0.0002993378502160653], "gradient_norm": 0.4086272716522217, "seconds": 0.31374972799994794, "nonpadding_tokens_per_second": 156385.7929464345, "ranks": [{"compute_seconds": 0.30660487899990585, "nonpadding_tokens": 1709, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30820536600003834, "nonpadding_tokens": 9737, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30974566499992306, "nonpadding_tokens": 3567, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3101288279999608, "nonpadding_tokens": 5307, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31048569600000064, "nonpadding_tokens": 5866, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30775153399997635, "nonpadding_tokens": 6652, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.304037266000023, "nonpadding_tokens": 9296, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3111459360000026, "nonpadding_tokens": 6932, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004215479999629679, "prefetch": {"data_wait_seconds": 0.017397039000982772, "consumed_batches": 1845, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2395, "tokens": 177112458, "loss_tokens": 46027, "loss": 2.077432811176049, "lr": 0.0002993374624021087, "optimizer_lrs": [0.019955830826807245, 0.0002993374624021087], "gradient_norm": 0.5215878486633301, "seconds": 0.26487735199998497, "nonpadding_tokens_per_second": 173767.21585468965, "ranks": [{"compute_seconds": 0.2618251170000576, "nonpadding_tokens": 3833, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25736030799998844, "nonpadding_tokens": 3540, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2602642570000171, "nonpadding_tokens": 2710, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2622823619999508, "nonpadding_tokens": 10417, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26126122799996665, "nonpadding_tokens": 5829, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2568899689999853, "nonpadding_tokens": 4710, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2610202630000913, "nonpadding_tokens": 12372, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.2543402109999988, "nonpadding_tokens": 2616, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003536330000315502, "prefetch": {"data_wait_seconds": 0.017405091000981088, "consumed_batches": 1846, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2396, "tokens": 177167655, "loss_tokens": 55197, "loss": 2.6046657313803285, "lr": 0.000299336997174283, "optimizer_lrs": [0.01995579981161887, 0.000299336997174283], "gradient_norm": 0.4848203659057617, "seconds": 0.2801954469999828, "nonpadding_tokens_per_second": 196994.63567658683, "ranks": [{"compute_seconds": 0.27639605500007747, "nonpadding_tokens": 9772, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2731605399999353, "nonpadding_tokens": 3515, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27481937799996103, "nonpadding_tokens": 3334, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2704825149999124, "nonpadding_tokens": 6507, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2775298330000169, "nonpadding_tokens": 17320, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2727034709999998, "nonpadding_tokens": 10812, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2751338750000514, "nonpadding_tokens": 2291, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.27611552999997, "nonpadding_tokens": 1646, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004148220000388392, "prefetch": {"data_wait_seconds": 0.01741472900103247, "consumed_batches": 1847, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2397, "tokens": 177261862, "loss_tokens": 94207, "loss": 2.1632065889477428, "lr": 0.00029933620277450675, "optimizer_lrs": [0.019955746851633784, 0.00029933620277450675], "gradient_norm": 0.4001200199127197, "seconds": 0.33045486699995763, "nonpadding_tokens_per_second": 285082.80375852983, "ranks": [{"compute_seconds": 0.3246565010000495, "nonpadding_tokens": 10715, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.322330715000021, "nonpadding_tokens": 17942, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3208290190000298, "nonpadding_tokens": 18400, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3257755610000004, "nonpadding_tokens": 8756, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32773526900007255, "nonpadding_tokens": 3911, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32387255700007245, "nonpadding_tokens": 12493, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3192212249999784, "nonpadding_tokens": 3583, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3264912469999217, "nonpadding_tokens": 18407, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004052389999742445, "prefetch": {"data_wait_seconds": 0.01742520200100728, "consumed_batches": 1848, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2398, "tokens": 177336882, "loss_tokens": 75020, "loss": 2.4750308251133033, "lr": 0.00029933556982960247, "optimizer_lrs": [0.019955704655306834, 0.00029933556982960247], "gradient_norm": 0.365292489528656, "seconds": 1.1666272710000385, "nonpadding_tokens_per_second": 64305.02857668713, "ranks": [{"compute_seconds": 1.1628163130000075, "nonpadding_tokens": 14736, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 1.1584178109999357, "nonpadding_tokens": 4032, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 1.1619534830000475, "nonpadding_tokens": 21664, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 1.1640210819999766, "nonpadding_tokens": 15651, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 1.1628039659999558, "nonpadding_tokens": 9043, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 1.157269018000079, "nonpadding_tokens": 3440, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 1.1625559870000188, "nonpadding_tokens": 1651, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 1.1548648709999725, "nonpadding_tokens": 4803, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042641200002435653, "prefetch": {"data_wait_seconds": 0.017435048001019027, "consumed_batches": 1849, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2399, "tokens": 177403580, "loss_tokens": 66698, "loss": 2.3598008935800174, "lr": 0.00029933500684487934, "optimizer_lrs": [0.019955667122991957, 0.00029933500684487934], "gradient_norm": 0.42026352882385254, "seconds": 0.29364828800009946, "nonpadding_tokens_per_second": 227135.66782305713, "ranks": [{"compute_seconds": 0.2313808119999976, "nonpadding_tokens": 15705, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23936859100001584, "nonpadding_tokens": 6708, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24179424699991614, "nonpadding_tokens": 1727, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2358467679999876, "nonpadding_tokens": 2588, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24198842099997364, "nonpadding_tokens": 7702, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2395117040000514, "nonpadding_tokens": 2539, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24186561000010443, "nonpadding_tokens": 17194, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.24239083600002687, "nonpadding_tokens": 12535, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.02614822299995012, "prefetch": {"data_wait_seconds": 0.01744613500098069, "consumed_batches": 1850, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2400, "tokens": 177469269, "loss_tokens": 65689, "loss": 2.525377155992632, "lr": 0.0002993344521444724, "optimizer_lrs": [0.01995563014296483, 0.0002993344521444724], "gradient_norm": 0.41385430097579956, "seconds": 0.37528965000001335, "nonpadding_tokens_per_second": 175035.46927019613, "ranks": [{"compute_seconds": 0.3725111440001001, "nonpadding_tokens": 12040, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3685463870000376, "nonpadding_tokens": 12716, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37072931200009407, "nonpadding_tokens": 3425, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3716009510000049, "nonpadding_tokens": 4958, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.372799600999997, "nonpadding_tokens": 8789, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3675917049999953, "nonpadding_tokens": 7819, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.365452925999989, "nonpadding_tokens": 12887, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.3725914940000621, "nonpadding_tokens": 3055, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004249270000400429, "prefetch": {"data_wait_seconds": 0.017455961000976572, "consumed_batches": 1851, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2401, "tokens": 177529657, "loss_tokens": 60388, "loss": 2.3852384786712593, "lr": 0.0002993339420040148, "optimizer_lrs": [0.01995559613360099, 0.0002993339420040148], "gradient_norm": 0.445625364780426, "seconds": 0.2988839460000463, "nonpadding_tokens_per_second": 202044.97701589714, "ranks": [{"compute_seconds": 0.2958516950000103, "nonpadding_tokens": 4584, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29193210799996905, "nonpadding_tokens": 4026, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29416718499999206, "nonpadding_tokens": 14459, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29486065000003236, "nonpadding_tokens": 3373, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29516815399995266, "nonpadding_tokens": 3196, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29262153000001945, "nonpadding_tokens": 2742, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2880440940000426, "nonpadding_tokens": 19275, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17630756864}, {"compute_seconds": 0.29637059499998486, "nonpadding_tokens": 8733, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003715499999543681, "prefetch": {"data_wait_seconds": 0.017465831000961884, "consumed_batches": 1852, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2402, "tokens": 177592938, "loss_tokens": 63281, "loss": 2.642797601175708, "lr": 0.00029933340721513996, "optimizer_lrs": [0.019955560481009334, 0.00029933340721513996], "gradient_norm": 0.45280975103378296, "seconds": 0.3204292279999663, "nonpadding_tokens_per_second": 197488.22663582565, "ranks": [{"compute_seconds": 0.31770327699996415, "nonpadding_tokens": 12955, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31375337400004355, "nonpadding_tokens": 5022, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31638609799995265, "nonpadding_tokens": 2694, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3167304739999963, "nonpadding_tokens": 10947, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3106563430000051, "nonpadding_tokens": 20459, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3131527509999614, "nonpadding_tokens": 6357, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31705769699999564, "nonpadding_tokens": 1101, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3179309680000415, "nonpadding_tokens": 3746, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003668350000225473, "prefetch": {"data_wait_seconds": 0.01747995900093713, "consumed_batches": 1853, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2403, "tokens": 177644484, "loss_tokens": 51546, "loss": 2.4032644991851937, "lr": 0.00029933297144075214, "optimizer_lrs": [0.01995553142938348, 0.00029933297144075214], "gradient_norm": 0.461305171251297, "seconds": 0.29394727500005047, "nonpadding_tokens_per_second": 175357.97873952446, "ranks": [{"compute_seconds": 0.2915429070000073, "nonpadding_tokens": 1510, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28775761899999, "nonpadding_tokens": 6559, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2906390719999763, "nonpadding_tokens": 6767, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29050656100002925, "nonpadding_tokens": 7631, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29100993299994116, "nonpadding_tokens": 6327, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2862450569999737, "nonpadding_tokens": 5537, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28401092399997196, "nonpadding_tokens": 12629, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2910796419999997, "nonpadding_tokens": 4586, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003947879999941506, "prefetch": {"data_wait_seconds": 0.017489702000901275, "consumed_batches": 1854, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2404, "tokens": 177697895, "loss_tokens": 53411, "loss": 2.3069896650502706, "lr": 0.0002993325197496486, "optimizer_lrs": [0.019955501316643243, 0.0002993325197496486], "gradient_norm": 0.4085029363632202, "seconds": 0.29640719300005003, "nonpadding_tokens_per_second": 180194.68238745132, "ranks": [{"compute_seconds": 0.29401459900009286, "nonpadding_tokens": 5828, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29027073600002495, "nonpadding_tokens": 7172, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2924541470000577, "nonpadding_tokens": 12469, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.287778000000003, "nonpadding_tokens": 6650, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2860935660000905, "nonpadding_tokens": 2630, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29075367600000845, "nonpadding_tokens": 4710, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2857350190000716, "nonpadding_tokens": 3132, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29371665200005737, "nonpadding_tokens": 10820, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037089199997808464, "prefetch": {"data_wait_seconds": 0.017498803000947305, "consumed_batches": 1855, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2405, "tokens": 177766230, "loss_tokens": 68335, "loss": 2.368135838150289, "lr": 0.00029933194162549926, "optimizer_lrs": [0.019955462775033284, 0.00029933194162549926], "gradient_norm": 0.3774028718471527, "seconds": 0.25591575799990096, "nonpadding_tokens_per_second": 267021.4625862408, "ranks": [{"compute_seconds": 0.2523064700000077, "nonpadding_tokens": 7390, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2493301030000339, "nonpadding_tokens": 4003, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25197484299997086, "nonpadding_tokens": 9605, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24711431799994443, "nonpadding_tokens": 16372, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24551417499992567, "nonpadding_tokens": 8775, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24880469200002153, "nonpadding_tokens": 3033, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2529000360000282, "nonpadding_tokens": 5216, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2534880499999872, "nonpadding_tokens": 13941, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003590999999687483, "prefetch": {"data_wait_seconds": 0.01750742400095362, "consumed_batches": 1856, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2406, "tokens": 177838666, "loss_tokens": 72436, "loss": 2.4846172397012536, "lr": 0.0002993313285337294, "optimizer_lrs": [0.01995542190224863, 0.0002993313285337294], "gradient_norm": 0.647965133190155, "seconds": 0.44237483199992766, "nonpadding_tokens_per_second": 163743.49253216974, "ranks": [{"compute_seconds": 0.4393521319999536, "nonpadding_tokens": 13903, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4361776319999535, "nonpadding_tokens": 3753, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.43834536400004254, "nonpadding_tokens": 7348, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.43707808799990744, "nonpadding_tokens": 4770, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4352341929999284, "nonpadding_tokens": 6673, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4365761349999957, "nonpadding_tokens": 12561, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4354704630000015, "nonpadding_tokens": 16288, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4397735530000091, "nonpadding_tokens": 7140, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003948229999650721, "prefetch": {"data_wait_seconds": 0.017517185000954782, "consumed_batches": 1857, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2407, "tokens": 177910385, "loss_tokens": 71719, "loss": 2.2448198437652507, "lr": 0.00029933072123423737, "optimizer_lrs": [0.019955381415615824, 0.00029933072123423737], "gradient_norm": 0.39977389574050903, "seconds": 0.32295710200003214, "nonpadding_tokens_per_second": 222069.74101468394, "ranks": [{"compute_seconds": 0.3162043880000738, "nonpadding_tokens": 11102, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3187168260000135, "nonpadding_tokens": 18461, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31477266899992173, "nonpadding_tokens": 5310, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31552476700005627, "nonpadding_tokens": 9931, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31416121600000224, "nonpadding_tokens": 9960, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31717926700002863, "nonpadding_tokens": 8204, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3202360709999539, "nonpadding_tokens": 2861, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3205792820000397, "nonpadding_tokens": 5890, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039334399991730606, "prefetch": {"data_wait_seconds": 0.01752518000098746, "consumed_batches": 1858, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2408, "tokens": 177980691, "loss_tokens": 70306, "loss": 2.61164520986829, "lr": 0.0002993301256327975, "optimizer_lrs": [0.01995534170885317, 0.0002993301256327975], "gradient_norm": 0.40205246210098267, "seconds": 0.27185198299991953, "nonpadding_tokens_per_second": 258618.67632586227, "ranks": [{"compute_seconds": 0.25570250500004477, "nonpadding_tokens": 1588, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.262161284000058, "nonpadding_tokens": 12105, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2591324610000356, "nonpadding_tokens": 8311, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2635327220000363, "nonpadding_tokens": 9678, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2584619599999769, "nonpadding_tokens": 12462, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2613271820000591, "nonpadding_tokens": 6050, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25835829999994075, "nonpadding_tokens": 16477, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2644776549999506, "nonpadding_tokens": 3635, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036162099991088326, "prefetch": {"data_wait_seconds": 0.01753424100100176, "consumed_batches": 1859, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2409, "tokens": 178024363, "loss_tokens": 43672, "loss": 2.441908574143616, "lr": 0.0002993297555298168, "optimizer_lrs": [0.01995531703532112, 0.0002993297555298168], "gradient_norm": 0.4721113443374634, "seconds": 0.2611980479999829, "nonpadding_tokens_per_second": 167198.7992804711, "ranks": [{"compute_seconds": 0.2552680320000036, "nonpadding_tokens": 3154, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25697668500004056, "nonpadding_tokens": 2713, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25752806900004543, "nonpadding_tokens": 4664, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.258048891000044, "nonpadding_tokens": 8997, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2584910839999566, "nonpadding_tokens": 6740, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25581742700001087, "nonpadding_tokens": 5048, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2527369379999982, "nonpadding_tokens": 3415, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25884462900000926, "nonpadding_tokens": 8941, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003537869999945542, "prefetch": {"data_wait_seconds": 0.01754355500095528, "consumed_batches": 1860, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2410, "tokens": 178128430, "loss_tokens": 104067, "loss": 2.3733904599921205, "lr": 0.0002993288731920721, "optimizer_lrs": [0.01995525821280481, 0.0002993288731920721], "gradient_norm": 0.5182897448539734, "seconds": 0.31220539200000985, "nonpadding_tokens_per_second": 333328.6441125806, "ranks": [{"compute_seconds": 0.30677012499995726, "nonpadding_tokens": 13626, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3061081880000529, "nonpadding_tokens": 5528, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30339998700003434, "nonpadding_tokens": 11133, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30902561200002765, "nonpadding_tokens": 14087, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3097378899999512, "nonpadding_tokens": 18771, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3052406009999231, "nonpadding_tokens": 12819, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3092206389999319, "nonpadding_tokens": 16215, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3030816110000387, "nonpadding_tokens": 11888, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003682590000835262, "prefetch": {"data_wait_seconds": 0.017551486000982663, "consumed_batches": 1861, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2411, "tokens": 178194714, "loss_tokens": 66284, "loss": 2.277706064057691, "lr": 0.0002993283108977466, "optimizer_lrs": [0.01995522072651644, 0.0002993283108977466], "gradient_norm": 0.42690542340278625, "seconds": 0.2542436340000904, "nonpadding_tokens_per_second": 260710.55922673142, "ranks": [{"compute_seconds": 0.2480907850000449, "nonpadding_tokens": 1332, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24762527199993656, "nonpadding_tokens": 12783, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24714500499999303, "nonpadding_tokens": 4845, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25116261799996664, "nonpadding_tokens": 13222, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2512410639999416, "nonpadding_tokens": 11693, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2485644629999797, "nonpadding_tokens": 4185, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24571984300007443, "nonpadding_tokens": 3733, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25181110000005447, "nonpadding_tokens": 14491, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000385712000024796, "prefetch": {"data_wait_seconds": 0.017560799001060445, "consumed_batches": 1862, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2412, "tokens": 178240545, "loss_tokens": 45831, "loss": 2.2423983630075712, "lr": 0.00029932792197108585, "optimizer_lrs": [0.019955194798072392, 0.00029932792197108585], "gradient_norm": 0.4393557906150818, "seconds": 0.27232491400002345, "nonpadding_tokens_per_second": 168295.2886198141, "ranks": [{"compute_seconds": 0.2698156950000339, "nonpadding_tokens": 5000, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26905513999997765, "nonpadding_tokens": 12548, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2698548050000227, "nonpadding_tokens": 1758, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26961506500003907, "nonpadding_tokens": 3528, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2653176909999502, "nonpadding_tokens": 7469, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26846704900003715, "nonpadding_tokens": 8089, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26981027300007554, "nonpadding_tokens": 4274, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.265486047999957, "nonpadding_tokens": 3165, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004404750000048807, "prefetch": {"data_wait_seconds": 0.017570305001143538, "consumed_batches": 1863, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2413, "tokens": 178313307, "loss_tokens": 72762, "loss": 2.3512577822214893, "lr": 0.0002993273042744985, "optimizer_lrs": [0.0199551536182999, 0.0002993273042744985], "gradient_norm": 0.46139785647392273, "seconds": 0.2641391779999367, "nonpadding_tokens_per_second": 275468.4123383523, "ranks": [{"compute_seconds": 0.2616013809999913, "nonpadding_tokens": 2000, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26129317300001276, "nonpadding_tokens": 5628, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2605830779999678, "nonpadding_tokens": 10097, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26089631100001043, "nonpadding_tokens": 16316, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2573602509999091, "nonpadding_tokens": 14125, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2600634860000355, "nonpadding_tokens": 6314, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25739019200000257, "nonpadding_tokens": 4094, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25751740800001244, "nonpadding_tokens": 14188, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043514000003597175, "prefetch": {"data_wait_seconds": 0.017578619001099582, "consumed_batches": 1864, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2414, "tokens": 178379206, "loss_tokens": 65899, "loss": 2.030404482617339, "lr": 0.00029932674459556623, "optimizer_lrs": [0.019955116306371083, 0.00029932674459556623], "gradient_norm": 1.467072606086731, "seconds": 0.30152281499999845, "nonpadding_tokens_per_second": 218553.94259303508, "ranks": [{"compute_seconds": 0.2952453530000412, "nonpadding_tokens": 5059, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29512541899998723, "nonpadding_tokens": 2550, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29765166999993653, "nonpadding_tokens": 25149, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.294605391999994, "nonpadding_tokens": 8501, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2929464789999656, "nonpadding_tokens": 2388, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2959244830000216, "nonpadding_tokens": 7537, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2988071439999658, "nonpadding_tokens": 6120, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29920630200001597, "nonpadding_tokens": 8595, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000352947999999742, "prefetch": {"data_wait_seconds": 0.017587937001124487, "consumed_batches": 1865, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2415, "tokens": 178440617, "loss_tokens": 61411, "loss": 2.582035384540229, "lr": 0.000299326222824162, "optimizer_lrs": [0.019955081521610803, 0.000299326222824162], "gradient_norm": 0.7500444054603577, "seconds": 0.38504314700003306, "nonpadding_tokens_per_second": 159491.21670770764, "ranks": [{"compute_seconds": 0.3823031890000266, "nonpadding_tokens": 18103, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3812003799999957, "nonpadding_tokens": 2071, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3811994129999903, "nonpadding_tokens": 13668, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.38130290000003697, "nonpadding_tokens": 3366, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3770415389999471, "nonpadding_tokens": 7947, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3803844300000492, "nonpadding_tokens": 4097, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3771750759999577, "nonpadding_tokens": 8367, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3826319960001001, "nonpadding_tokens": 3792, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003822329999820795, "prefetch": {"data_wait_seconds": 0.01759697600114407, "consumed_batches": 1866, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2416, "tokens": 178490217, "loss_tokens": 49600, "loss": 2.3755363218245966, "lr": 0.0002993258012563726, "optimizer_lrs": [0.019955053417091508, 0.0002993258012563726], "gradient_norm": 0.4290117919445038, "seconds": 0.20800730999997086, "nonpadding_tokens_per_second": 238453.15820875214, "ranks": [{"compute_seconds": 0.20555989099989347, "nonpadding_tokens": 5736, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20396141000003354, "nonpadding_tokens": 3109, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2049493080000957, "nonpadding_tokens": 10191, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20178221499998017, "nonpadding_tokens": 4600, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1996545890000334, "nonpadding_tokens": 6300, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20247213600009673, "nonpadding_tokens": 4705, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20532824999997956, "nonpadding_tokens": 5454, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20041085000002568, "nonpadding_tokens": 9505, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003448209999987739, "prefetch": {"data_wait_seconds": 0.01760595100115836, "consumed_batches": 1867, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2417, "tokens": 178553227, "loss_tokens": 63010, "loss": 2.314360815743533, "lr": 0.00029932526552264, "optimizer_lrs": [0.019955017701509332, 0.00029932526552264], "gradient_norm": 0.41859856247901917, "seconds": 0.3527971109999726, "nonpadding_tokens_per_second": 178601.23576806983, "ranks": [{"compute_seconds": 0.3493576870000652, "nonpadding_tokens": 11654, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3458912410000039, "nonpadding_tokens": 13282, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3488696430000573, "nonpadding_tokens": 7393, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3451436619999413, "nonpadding_tokens": 2031, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3499613540000155, "nonpadding_tokens": 3291, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34733230699998785, "nonpadding_tokens": 10027, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3497239040000295, "nonpadding_tokens": 8929, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3503642480000053, "nonpadding_tokens": 6403, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039533899996513355, "prefetch": {"data_wait_seconds": 0.017614931001233458, "consumed_batches": 1868, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2418, "tokens": 178618196, "loss_tokens": 64969, "loss": 2.414467621096215, "lr": 0.00029932471291054755, "optimizer_lrs": [0.019954980860703173, 0.00029932471291054755], "gradient_norm": 0.4162771701812744, "seconds": 0.2544365749999997, "nonpadding_tokens_per_second": 255344.57850645125, "ranks": [{"compute_seconds": 0.25122431299996606, "nonpadding_tokens": 4415, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2485496660000308, "nonpadding_tokens": 5584, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24602809199996045, "nonpadding_tokens": 1788, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24627664699994511, "nonpadding_tokens": 5040, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25199606599994695, "nonpadding_tokens": 8229, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2474605379999275, "nonpadding_tokens": 14046, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24503215499998987, "nonpadding_tokens": 12226, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2518214489999764, "nonpadding_tokens": 13641, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003455199999962133, "prefetch": {"data_wait_seconds": 0.0176227620012952, "consumed_batches": 1869, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2419, "tokens": 178720968, "loss_tokens": 102772, "loss": 1.9582002515276535, "lr": 0.000299323838293646, "optimizer_lrs": [0.019954922552909732, 0.000299323838293646], "gradient_norm": 1.1333949565887451, "seconds": 0.4221449140000004, "nonpadding_tokens_per_second": 243451.944087617, "ranks": [{"compute_seconds": 0.41677296800003205, "nonpadding_tokens": 18279, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4137102740000955, "nonpadding_tokens": 13593, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.41071367300003203, "nonpadding_tokens": 9807, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4114729039999929, "nonpadding_tokens": 4071, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.41653302800000347, "nonpadding_tokens": 21540, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4136556709999013, "nonpadding_tokens": 15933, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4160114750000048, "nonpadding_tokens": 10727, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.419489121999959, "nonpadding_tokens": 8822, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038454000002730027, "prefetch": {"data_wait_seconds": 0.017631329001304152, "consumed_batches": 1870, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2420, "tokens": 178753924, "loss_tokens": 32956, "loss": 2.4687262941497754, "lr": 0.00029932355770981436, "optimizer_lrs": [0.01995490384732096, 0.00029932355770981436], "gradient_norm": 0.6930726766586304, "seconds": 0.23988606800003254, "nonpadding_tokens_per_second": 137381.88413674582, "ranks": [{"compute_seconds": 0.21503474200005712, "nonpadding_tokens": 2240, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21566856299989468, "nonpadding_tokens": 3034, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.212596695000002, "nonpadding_tokens": 6560, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21330150699998285, "nonpadding_tokens": 6587, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21223829500002012, "nonpadding_tokens": 4386, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21683295900004396, "nonpadding_tokens": 4138, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2192985870000257, "nonpadding_tokens": 2786, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22083647900001324, "nonpadding_tokens": 3225, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.002834183999993911, "prefetch": {"data_wait_seconds": 0.017640221001286704, "consumed_batches": 1871, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2421, "tokens": 178815916, "loss_tokens": 61992, "loss": 2.702035645728481, "lr": 0.0002993230297592884, "optimizer_lrs": [0.01995486865061923, 0.0002993230297592884], "gradient_norm": 0.4274257719516754, "seconds": 0.25218168000003516, "nonpadding_tokens_per_second": 245822.77348612857, "ranks": [{"compute_seconds": 0.24966625500007922, "nonpadding_tokens": 4045, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24663034499997138, "nonpadding_tokens": 2388, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2436818600000379, "nonpadding_tokens": 5188, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24885433500003273, "nonpadding_tokens": 10422, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2495323760000474, "nonpadding_tokens": 3075, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24659825699995963, "nonpadding_tokens": 3650, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2420668420000993, "nonpadding_tokens": 28949, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24304202700000133, "nonpadding_tokens": 4275, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003591670000560043, "prefetch": {"data_wait_seconds": 0.01764926700123226, "consumed_batches": 1872, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2422, "tokens": 178868223, "loss_tokens": 52307, "loss": 2.3643910590360755, "lr": 0.0002993225841305913, "optimizer_lrs": [0.01995483894203942, 0.0002993225841305913], "gradient_norm": 0.4272618889808655, "seconds": 0.3981966310000189, "nonpadding_tokens_per_second": 131359.72514041065, "ranks": [{"compute_seconds": 0.3626168220000636, "nonpadding_tokens": 2488, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3598853069998995, "nonpadding_tokens": 5475, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35692208600005415, "nonpadding_tokens": 6390, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3575968229999944, "nonpadding_tokens": 5072, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35631227899989426, "nonpadding_tokens": 10251, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35860972199998287, "nonpadding_tokens": 10233, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36256995199994435, "nonpadding_tokens": 3164, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.36334791400008726, "nonpadding_tokens": 9234, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.004516623999961666, "prefetch": {"data_wait_seconds": 0.01765882300124133, "consumed_batches": 1873, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2423, "tokens": 178947587, "loss_tokens": 79364, "loss": 2.4224931959074643, "lr": 0.00029932190771088943, "optimizer_lrs": [0.01995479384739263, 0.00029932190771088943], "gradient_norm": 0.5651939511299133, "seconds": 0.3273656299999175, "nonpadding_tokens_per_second": 242432.29199112934, "ranks": [{"compute_seconds": 0.3202748950000114, "nonpadding_tokens": 14256, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32089580199999546, "nonpadding_tokens": 5041, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.323438603999989, "nonpadding_tokens": 4841, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32440837400008604, "nonpadding_tokens": 10530, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3175559369999519, "nonpadding_tokens": 13051, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32166536900001574, "nonpadding_tokens": 2058, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31747776700001396, "nonpadding_tokens": 10191, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3247804420000193, "nonpadding_tokens": 19396, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003588839999792981, "prefetch": {"data_wait_seconds": 0.017668182001173705, "consumed_batches": 1874, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2424, "tokens": 179023374, "loss_tokens": 75787, "loss": 2.3698465765896524, "lr": 0.0002993212614637002, "optimizer_lrs": [0.019954750764246684, 0.0002993212614637002], "gradient_norm": 0.4647084176540375, "seconds": 0.36098179900000105, "nonpadding_tokens_per_second": 209946.8732494177, "ranks": [{"compute_seconds": 0.3583690610000758, "nonpadding_tokens": 11969, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3551010879999694, "nonpadding_tokens": 14464, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3515828239999337, "nonpadding_tokens": 3939, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35836757199990643, "nonpadding_tokens": 8387, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35837531699996816, "nonpadding_tokens": 17732, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35732164700004887, "nonpadding_tokens": 3353, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35073307499999373, "nonpadding_tokens": 3249, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3511417160000292, "nonpadding_tokens": 12694, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044442000000799453, "prefetch": {"data_wait_seconds": 0.017678351001066517, "consumed_batches": 1875, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2425, "tokens": 179080653, "loss_tokens": 57279, "loss": 2.082380486303881, "lr": 0.000299320772833308, "optimizer_lrs": [0.0199547181888872, 0.000299320772833308], "gradient_norm": 0.4058822691440582, "seconds": 0.3155465520000007, "nonpadding_tokens_per_second": 181523.13703621097, "ranks": [{"compute_seconds": 0.3128093229998967, "nonpadding_tokens": 10170, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30971315900001173, "nonpadding_tokens": 10276, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30701045900002555, "nonpadding_tokens": 4390, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3074096240000017, "nonpadding_tokens": 6146, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3130492550000099, "nonpadding_tokens": 5341, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30911737600001743, "nonpadding_tokens": 3558, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3057685080000283, "nonpadding_tokens": 5865, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3064686210000218, "nonpadding_tokens": 11533, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034842299999127135, "prefetch": {"data_wait_seconds": 0.017687245001027918, "consumed_batches": 1876, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2426, "tokens": 179251263, "loss_tokens": 170610, "loss": 2.1844750087919818, "lr": 0.0002993193163701525, "optimizer_lrs": [0.0199546210913435, 0.0002993193163701525], "gradient_norm": 0.8395699262619019, "seconds": 0.32132132200001706, "nonpadding_tokens_per_second": 530963.8306541977, "ranks": [{"compute_seconds": 0.31462382300003355, "nonpadding_tokens": 9832, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3145930660000431, "nonpadding_tokens": 22564, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31282066500000383, "nonpadding_tokens": 27560, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31847111299998687, "nonpadding_tokens": 19619, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3187887059999639, "nonpadding_tokens": 29370, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31572075699989455, "nonpadding_tokens": 10712, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3114702099999249, "nonpadding_tokens": 24847, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.311968051000008, "nonpadding_tokens": 26106, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039010099999359227, "prefetch": {"data_wait_seconds": 0.01769610200103955, "consumed_batches": 1877, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2427, "tokens": 179352487, "loss_tokens": 101224, "loss": 2.2503849754010905, "lr": 0.0002993184515058595, "optimizer_lrs": [0.019954563433723968, 0.0002993184515058595], "gradient_norm": 0.8719368577003479, "seconds": 0.33071037699994577, "nonpadding_tokens_per_second": 306080.5074163627, "ranks": [{"compute_seconds": 0.3282106770000155, "nonpadding_tokens": 24399, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32557040000006054, "nonpadding_tokens": 20710, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32246384699999453, "nonpadding_tokens": 14154, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32818974999997863, "nonpadding_tokens": 5089, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3214511710000352, "nonpadding_tokens": 11356, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3260961390000148, "nonpadding_tokens": 2621, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32814410800006044, "nonpadding_tokens": 17027, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3220163419999835, "nonpadding_tokens": 5868, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038383399999020185, "prefetch": {"data_wait_seconds": 0.01770514900097453, "consumed_batches": 1878, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2428, "tokens": 179425283, "loss_tokens": 72796, "loss": 2.3967326072174293, "lr": 0.0002993178291936322, "optimizer_lrs": [0.01995452194624215, 0.0002993178291936322], "gradient_norm": 0.49946677684783936, "seconds": 0.31762795300005564, "nonpadding_tokens_per_second": 229186.37768630913, "ranks": [{"compute_seconds": 0.30938222899999346, "nonpadding_tokens": 5646, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31114976099991054, "nonpadding_tokens": 5395, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3082960329999196, "nonpadding_tokens": 4821, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31402490400000715, "nonpadding_tokens": 3834, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31469873999992615, "nonpadding_tokens": 22017, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31027053499997237, "nonpadding_tokens": 6417, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30780019799999536, "nonpadding_tokens": 14880, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3151517149999563, "nonpadding_tokens": 9786, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003983459999972183, "prefetch": {"data_wait_seconds": 0.017714657000922784, "consumed_batches": 1879, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2429, "tokens": 179468832, "loss_tokens": 43549, "loss": 2.5829695788077798, "lr": 0.00029931745677024425, "optimizer_lrs": [0.019954497118016286, 0.00029931745677024425], "gradient_norm": 0.48070135712623596, "seconds": 0.26909365599999546, "nonpadding_tokens_per_second": 161835.84796217096, "ranks": [{"compute_seconds": 0.2667438889999403, "nonpadding_tokens": 4893, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26583232999996653, "nonpadding_tokens": 8111, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26609347299995534, "nonpadding_tokens": 2042, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26332269099998484, "nonpadding_tokens": 7572, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2618384419999984, "nonpadding_tokens": 11167, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26397630899998603, "nonpadding_tokens": 4188, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2616859290000093, "nonpadding_tokens": 4123, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26233721300002344, "nonpadding_tokens": 1453, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035006100006285124, "prefetch": {"data_wait_seconds": 0.01772325000088415, "consumed_batches": 1880, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2430, "tokens": 179538142, "loss_tokens": 69310, "loss": 2.3800132556629636, "lr": 0.00029931686383425336, "optimizer_lrs": [0.019954457588950224, 0.00029931686383425336], "gradient_norm": 0.8857668042182922, "seconds": 0.3535421099999212, "nonpadding_tokens_per_second": 196044.5390791367, "ranks": [{"compute_seconds": 0.3204904479999868, "nonpadding_tokens": 11142, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3233479929999703, "nonpadding_tokens": 12356, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32412086899989845, "nonpadding_tokens": 7227, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32460611499993774, "nonpadding_tokens": 9016, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31937886700006857, "nonpadding_tokens": 3699, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32209815700002764, "nonpadding_tokens": 4101, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3188984439999558, "nonpadding_tokens": 11523, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3255094509999026, "nonpadding_tokens": 10246, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.022652354999991076, "prefetch": {"data_wait_seconds": 0.017732979000882665, "consumed_batches": 1881, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2431, "tokens": 179591585, "loss_tokens": 53443, "loss": 2.624659099414329, "lr": 0.00029931640646258233, "optimizer_lrs": [0.01995442709750549, 0.00029931640646258233], "gradient_norm": 0.48739877343177795, "seconds": 0.3194478180000715, "nonpadding_tokens_per_second": 167298.0593030316, "ranks": [{"compute_seconds": 0.3164437329999146, "nonpadding_tokens": 2483, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3134420029999774, "nonpadding_tokens": 5699, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31668285199998536, "nonpadding_tokens": 2003, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3167676210000536, "nonpadding_tokens": 5858, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3107967710000139, "nonpadding_tokens": 5709, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3142431480000596, "nonpadding_tokens": 20700, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3165842870000688, "nonpadding_tokens": 9023, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3155295220000198, "nonpadding_tokens": 1968, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004920869999978095, "prefetch": {"data_wait_seconds": 0.017744075000862836, "consumed_batches": 1882, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2432, "tokens": 179667940, "loss_tokens": 76355, "loss": 2.514549022657324, "lr": 0.00029931575274236077, "optimizer_lrs": [0.019954383516157387, 0.00029931575274236077], "gradient_norm": 0.4586547613143921, "seconds": 0.30076691700003266, "nonpadding_tokens_per_second": 253867.68186340024, "ranks": [{"compute_seconds": 0.298007681999934, "nonpadding_tokens": 6367, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29463100600003145, "nonpadding_tokens": 18640, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.296751507999943, "nonpadding_tokens": 12762, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29264885699990373, "nonpadding_tokens": 7848, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.291232358000002, "nonpadding_tokens": 13309, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29557988100009425, "nonpadding_tokens": 2659, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2911137480000434, "nonpadding_tokens": 10159, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2981718670000646, "nonpadding_tokens": 4611, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039969099998415913, "prefetch": {"data_wait_seconds": 0.017754103000882537, "consumed_batches": 1883, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2433, "tokens": 179749775, "loss_tokens": 81835, "loss": 2.4358722963890758, "lr": 0.0002993150517586407, "optimizer_lrs": [0.01995433678390938, 0.0002993150517586407], "gradient_norm": 0.433206170797348, "seconds": 0.5038919510000142, "nonpadding_tokens_per_second": 162405.84878879657, "ranks": [{"compute_seconds": 0.5003145739999582, "nonpadding_tokens": 6068, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4974772230000326, "nonpadding_tokens": 10402, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4972348090000196, "nonpadding_tokens": 13015, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.5013014580000572, "nonpadding_tokens": 7847, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.49661756900002274, "nonpadding_tokens": 8142, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.49819012099999327, "nonpadding_tokens": 11392, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.496044119999965, "nonpadding_tokens": 15398, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.5014267059999611, "nonpadding_tokens": 9571, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004068189999770766, "prefetch": {"data_wait_seconds": 0.017764381000986873, "consumed_batches": 1884, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2434, "tokens": 179817222, "loss_tokens": 67447, "loss": 2.4987341912909398, "lr": 0.0002993144737508155, "optimizer_lrs": [0.01995429825005437, 0.0002993144737508155], "gradient_norm": 0.5167602896690369, "seconds": 0.2419778010000755, "nonpadding_tokens_per_second": 278732.18006464554, "ranks": [{"compute_seconds": 0.23804191000010633, "nonpadding_tokens": 7361, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2362108840000019, "nonpadding_tokens": 5509, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23802509300003294, "nonpadding_tokens": 2955, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2388792279999734, "nonpadding_tokens": 4934, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23272280699995918, "nonpadding_tokens": 10304, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23598633100004918, "nonpadding_tokens": 3926, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23253884900009325, "nonpadding_tokens": 7266, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23949339700004657, "nonpadding_tokens": 25192, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036561900003562187, "prefetch": {"data_wait_seconds": 0.01777342300097473, "consumed_batches": 1885, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2435, "tokens": 179888049, "loss_tokens": 70827, "loss": 2.333617255778164, "lr": 0.0002993138665152982, "optimizer_lrs": [0.01995425776768655, 0.0002993138665152982], "gradient_norm": 0.3856537938117981, "seconds": 0.24379213899999286, "nonpadding_tokens_per_second": 290522.0828305792, "ranks": [{"compute_seconds": 0.2398672340000303, "nonpadding_tokens": 3696, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23707287599995652, "nonpadding_tokens": 14029, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2395922859999473, "nonpadding_tokens": 26025, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23832873499998186, "nonpadding_tokens": 8376, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23594798099998116, "nonpadding_tokens": 2892, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23931450400004906, "nonpadding_tokens": 1640, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24031548999994357, "nonpadding_tokens": 6351, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24135373399997206, "nonpadding_tokens": 7818, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037819200008470943, "prefetch": {"data_wait_seconds": 0.017782948000970578, "consumed_batches": 1886, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2436, "tokens": 179947177, "loss_tokens": 59128, "loss": 2.4549954124949265, "lr": 0.0002993133593758347, "optimizer_lrs": [0.019954223958388983, 0.0002993133593758347], "gradient_norm": 0.4264031946659088, "seconds": 0.34692899499998475, "nonpadding_tokens_per_second": 170432.56935040152, "ranks": [{"compute_seconds": 0.34368830400001116, "nonpadding_tokens": 2847, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34277658000007705, "nonpadding_tokens": 15113, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34323821899999984, "nonpadding_tokens": 5353, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3406472120000217, "nonpadding_tokens": 1902, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3442687780000142, "nonpadding_tokens": 7406, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3416595740000048, "nonpadding_tokens": 5953, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3398144660000071, "nonpadding_tokens": 11454, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34452250100002857, "nonpadding_tokens": 9100, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003777170001058039, "prefetch": {"data_wait_seconds": 0.01779343300093217, "consumed_batches": 1887, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2437, "tokens": 180031488, "loss_tokens": 84311, "loss": 2.2641646330253464, "lr": 0.0002993126359191657, "optimizer_lrs": [0.019954175727944383, 0.0002993126359191657], "gradient_norm": 0.40414100885391235, "seconds": 0.33489035999991756, "nonpadding_tokens_per_second": 251757.02280597374, "ranks": [{"compute_seconds": 0.3319409910000104, "nonpadding_tokens": 11261, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33142426099993827, "nonpadding_tokens": 15698, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3285210310000366, "nonpadding_tokens": 11415, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33132731500006685, "nonpadding_tokens": 1830, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3275482330000159, "nonpadding_tokens": 9965, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33043406799993136, "nonpadding_tokens": 8741, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32776886900001045, "nonpadding_tokens": 8182, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33231290900005206, "nonpadding_tokens": 17219, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0006476539999766828, "prefetch": {"data_wait_seconds": 0.017803752000943973, "consumed_batches": 1888, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2438, "tokens": 180140053, "loss_tokens": 108565, "loss": 1.3985545813567908, "lr": 0.0002993117037839086, "optimizer_lrs": [0.01995411358559391, 0.0002993117037839086], "gradient_norm": 0.6081315875053406, "seconds": 0.3621614219999856, "nonpadding_tokens_per_second": 299769.64249937236, "ranks": [{"compute_seconds": 0.35751157999993666, "nonpadding_tokens": 14426, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35610969399999703, "nonpadding_tokens": 21482, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35818111800006136, "nonpadding_tokens": 11356, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3536558299999797, "nonpadding_tokens": 11268, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3523449739999478, "nonpadding_tokens": 9399, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3565697059999593, "nonpadding_tokens": 12809, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35900840099998277, "nonpadding_tokens": 17743, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35945513099989057, "nonpadding_tokens": 10082, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004141240000308244, "prefetch": {"data_wait_seconds": 0.0178135070008949, "consumed_batches": 1889, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2439, "tokens": 180204960, "loss_tokens": 64907, "loss": 1.768941042568598, "lr": 0.0002993111461937815, "optimizer_lrs": [0.019954076412918766, 0.0002993111461937815], "gradient_norm": 0.5081087946891785, "seconds": 0.23652451800001018, "nonpadding_tokens_per_second": 274419.75381172623, "ranks": [{"compute_seconds": 0.22884268599989355, "nonpadding_tokens": 7568, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23064658500004498, "nonpadding_tokens": 9999, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23217669699999988, "nonpadding_tokens": 18127, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22798045999991245, "nonpadding_tokens": 5104, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23342525900000055, "nonpadding_tokens": 3581, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23032582599989837, "nonpadding_tokens": 10384, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22680230200001006, "nonpadding_tokens": 5335, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23356587199998557, "nonpadding_tokens": 4809, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037571300003946817, "prefetch": {"data_wait_seconds": 0.017822908000951543, "consumed_batches": 1890, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2440, "tokens": 180277635, "loss_tokens": 72675, "loss": 2.393421482628139, "lr": 0.0002993105216046467, "optimizer_lrs": [0.019954034773643114, 0.0002993105216046467], "gradient_norm": 0.4512934386730194, "seconds": 0.35180601900003694, "nonpadding_tokens_per_second": 206576.90907781874, "ranks": [{"compute_seconds": 0.34606606700003795, "nonpadding_tokens": 9402, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3460368340000741, "nonpadding_tokens": 6248, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34789467899997817, "nonpadding_tokens": 1658, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3436894230000007, "nonpadding_tokens": 11668, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34253397800000585, "nonpadding_tokens": 13697, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3463871559999916, "nonpadding_tokens": 2986, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3487211380000872, "nonpadding_tokens": 2258, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34934008799996263, "nonpadding_tokens": 24758, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038018699990516325, "prefetch": {"data_wait_seconds": 0.017832313000965883, "consumed_batches": 1891, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2441, "tokens": 180365162, "loss_tokens": 87527, "loss": 2.2885366801101372, "lr": 0.00029930976899855824, "optimizer_lrs": [0.019953984599903885, 0.00029930976899855824], "gradient_norm": 0.5886831879615784, "seconds": 0.3729405599999609, "nonpadding_tokens_per_second": 234694.23652929888, "ranks": [{"compute_seconds": 0.37061210800004574, "nonpadding_tokens": 26888, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36835802100006276, "nonpadding_tokens": 9686, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3689860369998996, "nonpadding_tokens": 6853, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3696939389999443, "nonpadding_tokens": 25778, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36465511599999445, "nonpadding_tokens": 3682, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3671087440000065, "nonpadding_tokens": 4162, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36452249599994957, "nonpadding_tokens": 6110, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3701818089999733, "nonpadding_tokens": 4368, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038436099998762074, "prefetch": {"data_wait_seconds": 0.01784160100096699, "consumed_batches": 1892, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2442, "tokens": 180465790, "loss_tokens": 100628, "loss": 2.292797792363954, "lr": 0.000299308903236743, "optimizer_lrs": [0.019953926882449537, 0.000299308903236743], "gradient_norm": 0.5063559412956238, "seconds": 0.2572015560000409, "nonpadding_tokens_per_second": 391241.8010409859, "ranks": [{"compute_seconds": 0.2543183100000306, "nonpadding_tokens": 7092, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25323035799999616, "nonpadding_tokens": 10552, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25084446099992874, "nonpadding_tokens": 15634, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25110922800001845, "nonpadding_tokens": 18050, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24951706100000592, "nonpadding_tokens": 11314, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25212892299998657, "nonpadding_tokens": 14157, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.254738388000078, "nonpadding_tokens": 11947, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25012334500002, "nonpadding_tokens": 11882, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004222559999789155, "prefetch": {"data_wait_seconds": 0.01785117300092054, "consumed_batches": 1893, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2443, "tokens": 180495861, "loss_tokens": 30071, "loss": 2.354027239699378, "lr": 0.000299308644413224, "optimizer_lrs": [0.019953909627548266, 0.000299308644413224], "gradient_norm": 0.46687576174736023, "seconds": 0.25635143400006655, "nonpadding_tokens_per_second": 117303.81036211482, "ranks": [{"compute_seconds": 0.2529527080000662, "nonpadding_tokens": 3802, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2512928790000615, "nonpadding_tokens": 3072, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24818444400000317, "nonpadding_tokens": 4178, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2538527780000095, "nonpadding_tokens": 3534, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24717770200004452, "nonpadding_tokens": 3854, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2514554179999777, "nonpadding_tokens": 3088, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2539526239999077, "nonpadding_tokens": 3845, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24777219000009154, "nonpadding_tokens": 4698, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004107349999458165, "prefetch": {"data_wait_seconds": 0.017860489000852908, "consumed_batches": 1894, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2444, "tokens": 180562291, "loss_tokens": 66430, "loss": 2.5837968538311005, "lr": 0.00029930807247354244, "optimizer_lrs": [0.019953871498236166, 0.00029930807247354244], "gradient_norm": 0.5313018560409546, "seconds": 0.32149298399997406, "nonpadding_tokens_per_second": 206629.7036205473, "ranks": [{"compute_seconds": 0.2737372360001018, "nonpadding_tokens": 3495, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3180488670000159, "nonpadding_tokens": 2479, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3180224969999017, "nonpadding_tokens": 12248, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31889332000002923, "nonpadding_tokens": 16701, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31392844999993486, "nonpadding_tokens": 12642, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3165452649999452, "nonpadding_tokens": 847, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31889327299995784, "nonpadding_tokens": 14162, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31443323500002407, "nonpadding_tokens": 3856, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004030959998999606, "prefetch": {"data_wait_seconds": 0.017869602000772034, "consumed_batches": 1895, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2445, "tokens": 180642360, "loss_tokens": 80069, "loss": 2.358631305499007, "lr": 0.0002993073827932331, "optimizer_lrs": [0.019953825519548874, 0.0002993073827932331], "gradient_norm": 0.46184906363487244, "seconds": 0.24738407500001358, "nonpadding_tokens_per_second": 323662.7094933075, "ranks": [{"compute_seconds": 0.2408576830000584, "nonpadding_tokens": 3814, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24098298299998078, "nonpadding_tokens": 18842, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23790589399993678, "nonpadding_tokens": 23722, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23869154099998013, "nonpadding_tokens": 8310, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23685054500003844, "nonpadding_tokens": 5773, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2415057360000219, "nonpadding_tokens": 11857, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24490760800006228, "nonpadding_tokens": 4766, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23747621700010768, "nonpadding_tokens": 2985, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004323160000012649, "prefetch": {"data_wait_seconds": 0.01788144500081046, "consumed_batches": 1896, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2446, "tokens": 180687285, "loss_tokens": 44925, "loss": 2.3135496661101835, "lr": 0.0002993069956783234, "optimizer_lrs": [0.01995379971188823, 0.0002993069956783234], "gradient_norm": 0.42966336011886597, "seconds": 0.23974153199992543, "nonpadding_tokens_per_second": 187389.30891629562, "ranks": [{"compute_seconds": 0.23411305000001903, "nonpadding_tokens": 3861, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2333771420001085, "nonpadding_tokens": 3370, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.231779897000024, "nonpadding_tokens": 4017, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2366438599999583, "nonpadding_tokens": 4335, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23031418499999745, "nonpadding_tokens": 3215, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23324455800002397, "nonpadding_tokens": 4295, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2366250449999825, "nonpadding_tokens": 8349, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23733892299992476, "nonpadding_tokens": 13483, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040563799996107264, "prefetch": {"data_wait_seconds": 0.017891745000795822, "consumed_batches": 1897, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2447, "tokens": 180751510, "loss_tokens": 64225, "loss": 2.534408330089529, "lr": 0.00029930644206960986, "optimizer_lrs": [0.01995376280464066, 0.00029930644206960986], "gradient_norm": 0.5013068914413452, "seconds": 0.2612437149999778, "nonpadding_tokens_per_second": 245843.2349272229, "ranks": [{"compute_seconds": 0.2586237670000173, "nonpadding_tokens": 4698, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25894554399997105, "nonpadding_tokens": 3304, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25833613500003594, "nonpadding_tokens": 6070, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2565657850000207, "nonpadding_tokens": 12129, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2553378730000304, "nonpadding_tokens": 11495, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25760994300003404, "nonpadding_tokens": 3190, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2550635270000612, "nonpadding_tokens": 10871, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2555619529999831, "nonpadding_tokens": 12468, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003714040000204477, "prefetch": {"data_wait_seconds": 0.017901415000736165, "consumed_batches": 1898, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2448, "tokens": 180822903, "loss_tokens": 71393, "loss": 2.2411580967321725, "lr": 0.0002993058264151899, "optimizer_lrs": [0.01995372176101266, 0.0002993058264151899], "gradient_norm": 0.49085667729377747, "seconds": 0.3782768879999594, "nonpadding_tokens_per_second": 188732.1225927175, "ranks": [{"compute_seconds": 0.3730669530000341, "nonpadding_tokens": 15378, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36994881400005397, "nonpadding_tokens": 11660, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3703742949999196, "nonpadding_tokens": 13059, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.37311246899992057, "nonpadding_tokens": 12241, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.37361441199993806, "nonpadding_tokens": 10247, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3708425840000018, "nonpadding_tokens": 3874, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3735017180000568, "nonpadding_tokens": 3379, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.37565005699991616, "nonpadding_tokens": 1555, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040281500002947723, "prefetch": {"data_wait_seconds": 0.01791049100070552, "consumed_batches": 1899, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2449, "tokens": 180878321, "loss_tokens": 55418, "loss": 2.516731364358151, "lr": 0.0002993053483326774, "optimizer_lrs": [0.01995368988884516, 0.0002993053483326774], "gradient_norm": 0.5275440216064453, "seconds": 0.25790222599994195, "nonpadding_tokens_per_second": 214879.88242494842, "ranks": [{"compute_seconds": 0.25371393600005376, "nonpadding_tokens": 5109, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25454854600002363, "nonpadding_tokens": 15381, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2539732709999498, "nonpadding_tokens": 12483, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2545080639999924, "nonpadding_tokens": 2288, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25537444400004006, "nonpadding_tokens": 2531, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25355325900000025, "nonpadding_tokens": 7851, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2547153980000303, "nonpadding_tokens": 4105, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25545899899998403, "nonpadding_tokens": 5670, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003542020000395496, "prefetch": {"data_wait_seconds": 0.01791891600073825, "consumed_batches": 1900, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2450, "tokens": 180916225, "loss_tokens": 37904, "loss": 2.3037158842074716, "lr": 0.00029930502124620253, "optimizer_lrs": [0.01995366808308017, 0.00029930502124620253], "gradient_norm": 0.5823766589164734, "seconds": 0.23435532300004525, "nonpadding_tokens_per_second": 161737.31202168035, "ranks": [{"compute_seconds": 0.23196245500002988, "nonpadding_tokens": 1800, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23085202199990817, "nonpadding_tokens": 5105, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22774268299997402, "nonpadding_tokens": 5420, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2295202010000139, "nonpadding_tokens": 2466, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23127884199993787, "nonpadding_tokens": 4530, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22974233000002187, "nonpadding_tokens": 6908, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22697119099996144, "nonpadding_tokens": 6226, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22790811799995936, "nonpadding_tokens": 5449, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004341090000252734, "prefetch": {"data_wait_seconds": 0.017928964000773817, "consumed_batches": 1901, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2451, "tokens": 180977981, "loss_tokens": 61756, "loss": 2.394890020807695, "lr": 0.0002993044881682516, "optimizer_lrs": [0.01995363254455011, 0.0002993044881682516], "gradient_norm": 0.5551803708076477, "seconds": 0.39057044000003316, "nonpadding_tokens_per_second": 158117.44483272918, "ranks": [{"compute_seconds": 0.38784066999994593, "nonpadding_tokens": 20525, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3834854339999083, "nonpadding_tokens": 6108, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3815912649999973, "nonpadding_tokens": 4188, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3820195410000906, "nonpadding_tokens": 3102, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.38050963899991075, "nonpadding_tokens": 8632, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3846827250000615, "nonpadding_tokens": 2019, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.38802872799999477, "nonpadding_tokens": 6802, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.38100694199999907, "nonpadding_tokens": 10380, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003942629999755809, "prefetch": {"data_wait_seconds": 0.01793801400079076, "consumed_batches": 1902, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2452, "tokens": 181046008, "loss_tokens": 68027, "loss": 2.667568805400797, "lr": 0.0002993039007230731, "optimizer_lrs": [0.01995359338153821, 0.0002993039007230731], "gradient_norm": 0.48582127690315247, "seconds": 0.3236299579999695, "nonpadding_tokens_per_second": 210199.946940655, "ranks": [{"compute_seconds": 0.3206590639999831, "nonpadding_tokens": 4055, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31918103900000006, "nonpadding_tokens": 2962, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.315808365999942, "nonpadding_tokens": 17947, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32111886500001674, "nonpadding_tokens": 19140, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31499831799999356, "nonpadding_tokens": 7556, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3184453989999838, "nonpadding_tokens": 4557, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3210340539999379, "nonpadding_tokens": 4034, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3174399380000068, "nonpadding_tokens": 7776, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004092179999588552, "prefetch": {"data_wait_seconds": 0.0179474110008897, "consumed_batches": 1903, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2453, "tokens": 181100457, "loss_tokens": 54449, "loss": 2.3267716693603187, "lr": 0.00029930343035210487, "optimizer_lrs": [0.019953562023473658, 0.00029930343035210487], "gradient_norm": 0.4212595224380493, "seconds": 0.32131323099997644, "nonpadding_tokens_per_second": 169457.69656153372, "ranks": [{"compute_seconds": 0.31397728399997504, "nonpadding_tokens": 11925, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31499914000005447, "nonpadding_tokens": 9095, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31350713899996663, "nonpadding_tokens": 2791, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31789820299991334, "nonpadding_tokens": 12370, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31262948699998105, "nonpadding_tokens": 9534, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3163596320000579, "nonpadding_tokens": 3055, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31235322499992435, "nonpadding_tokens": 2279, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3189869010000166, "nonpadding_tokens": 3400, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003405920000432161, "prefetch": {"data_wait_seconds": 0.01795631400086961, "consumed_batches": 1904, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2454, "tokens": 181168859, "loss_tokens": 68402, "loss": 2.42622748603842, "lr": 0.00029930283922017576, "optimizer_lrs": [0.019953522614678386, 0.00029930283922017576], "gradient_norm": 0.4118395745754242, "seconds": 0.2887125089999927, "nonpadding_tokens_per_second": 236920.8048411984, "ranks": [{"compute_seconds": 0.28207842799997707, "nonpadding_tokens": 9537, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28290959100002055, "nonpadding_tokens": 15590, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2853677549999247, "nonpadding_tokens": 6046, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2817462579999983, "nonpadding_tokens": 2977, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2863351529999818, "nonpadding_tokens": 4961, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2828848280000784, "nonpadding_tokens": 16202, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2801008719999345, "nonpadding_tokens": 5375, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28095835699991767, "nonpadding_tokens": 7714, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039655900002344424, "prefetch": {"data_wait_seconds": 0.01796457100086002, "consumed_batches": 1905, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2455, "tokens": 181235128, "loss_tokens": 66269, "loss": 2.3088201591241755, "lr": 0.0002993022662831944, "optimizer_lrs": [0.019953484418879626, 0.0002993022662831944], "gradient_norm": 0.41795361042022705, "seconds": 0.2764021360000015, "nonpadding_tokens_per_second": 239755.74486877205, "ranks": [{"compute_seconds": 0.2667335710000316, "nonpadding_tokens": 2545, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26885635899998306, "nonpadding_tokens": 5393, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26536863699993773, "nonpadding_tokens": 6597, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27339029299992035, "nonpadding_tokens": 8616, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2734864059999609, "nonpadding_tokens": 11492, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26715679999995245, "nonpadding_tokens": 4025, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2731874500000231, "nonpadding_tokens": 14871, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27398043300001973, "nonpadding_tokens": 12730, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039663500001552165, "prefetch": {"data_wait_seconds": 0.017972477000810727, "consumed_batches": 1906, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2456, "tokens": 181318159, "loss_tokens": 83031, "loss": 2.455173557165396, "lr": 0.00029930154809695267, "optimizer_lrs": [0.019953436539796848, 0.00029930154809695267], "gradient_norm": 0.4169428050518036, "seconds": 0.2873864609999828, "nonpadding_tokens_per_second": 288917.5770879651, "ranks": [{"compute_seconds": 0.2847236160000648, "nonpadding_tokens": 8712, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2849369550000347, "nonpadding_tokens": 14873, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28168501100003596, "nonpadding_tokens": 10103, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2842114880000963, "nonpadding_tokens": 10408, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28506939800001874, "nonpadding_tokens": 6619, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2838090689999717, "nonpadding_tokens": 10872, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2807820159999892, "nonpadding_tokens": 14942, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28175579200001266, "nonpadding_tokens": 6502, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004015800000161107, "prefetch": {"data_wait_seconds": 0.01798143700086996, "consumed_batches": 1907, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2457, "tokens": 181391153, "loss_tokens": 72994, "loss": 2.1785596247636794, "lr": 0.0002993009164225537, "optimizer_lrs": [0.01995339442817025, 0.0002993009164225537], "gradient_norm": 0.3674013316631317, "seconds": 0.2801017589999901, "nonpadding_tokens_per_second": 260598.14926047134, "ranks": [{"compute_seconds": 0.27766226800008553, "nonpadding_tokens": 11731, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2753090879999718, "nonpadding_tokens": 10536, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27740563100007876, "nonpadding_tokens": 24989, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2778761839999788, "nonpadding_tokens": 3870, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2744680100000778, "nonpadding_tokens": 3751, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27577444400003515, "nonpadding_tokens": 2773, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27411940299998605, "nonpadding_tokens": 6051, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27498181700002533, "nonpadding_tokens": 9293, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035390300001836295, "prefetch": {"data_wait_seconds": 0.017990444000886896, "consumed_batches": 1908, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2458, "tokens": 181434954, "loss_tokens": 43801, "loss": 2.4480280844044655, "lr": 0.0002993005372413974, "optimizer_lrs": [0.019953369149426494, 0.0002993005372413974], "gradient_norm": 0.47672829031944275, "seconds": 0.2835989430000154, "nonpadding_tokens_per_second": 154446.97902134852, "ranks": [{"compute_seconds": 0.24961869300000217, "nonpadding_tokens": 5427, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2593834950000655, "nonpadding_tokens": 6202, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25860225400003856, "nonpadding_tokens": 2553, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2593957579999824, "nonpadding_tokens": 1573, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2597606510000787, "nonpadding_tokens": 3751, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2570851060000905, "nonpadding_tokens": 3233, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25975841399997535, "nonpadding_tokens": 9222, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26010826699996414, "nonpadding_tokens": 11840, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00469694299999901, "prefetch": {"data_wait_seconds": 0.017998592000893154, "consumed_batches": 1909, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2459, "tokens": 181521282, "loss_tokens": 86328, "loss": 2.5266612150750625, "lr": 0.0002992997896077811, "optimizer_lrs": [0.01995331930718541, 0.0002992997896077811], "gradient_norm": 0.46625474095344543, "seconds": 0.24799428200003604, "nonpadding_tokens_per_second": 348104.8002549811, "ranks": [{"compute_seconds": 0.24533644899997853, "nonpadding_tokens": 5845, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24480924900001355, "nonpadding_tokens": 18080, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24429862500005584, "nonpadding_tokens": 3543, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24273398499997256, "nonpadding_tokens": 13344, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24550371100008306, "nonpadding_tokens": 13362, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24389391600004728, "nonpadding_tokens": 9168, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24096326500000487, "nonpadding_tokens": 18060, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2456435800000918, "nonpadding_tokens": 4926, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003669420000278478, "prefetch": {"data_wait_seconds": 0.018008120000857275, "consumed_batches": 1910, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2460, "tokens": 181573714, "loss_tokens": 52432, "loss": 2.508879358025633, "lr": 0.0002992993353321126, "optimizer_lrs": [0.01995328902214084, 0.0002992993353321126], "gradient_norm": 0.4657834470272064, "seconds": 0.3973946659999683, "nonpadding_tokens_per_second": 131939.36528580426, "ranks": [{"compute_seconds": 0.3921898970000939, "nonpadding_tokens": 10909, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.39381537899998875, "nonpadding_tokens": 13364, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3914972359999638, "nonpadding_tokens": 5751, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3939237490000096, "nonpadding_tokens": 6112, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.39481295600000976, "nonpadding_tokens": 7628, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3917124580000291, "nonpadding_tokens": 3722, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.394314596000072, "nonpadding_tokens": 1734, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.39503118400000403, "nonpadding_tokens": 3212, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035401100001308805, "prefetch": {"data_wait_seconds": 0.018018110000824095, "consumed_batches": 1911, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2461, "tokens": 181626450, "loss_tokens": 52736, "loss": 2.332340870088744, "lr": 0.0002992988782743508, "optimizer_lrs": [0.019953258551623387, 0.0002992988782743508], "gradient_norm": 0.4301738440990448, "seconds": 0.22590929499995127, "nonpadding_tokens_per_second": 233438.82331185785, "ranks": [{"compute_seconds": 0.22075905399992735, "nonpadding_tokens": 6884, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22316532199999983, "nonpadding_tokens": 7160, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22011091499996382, "nonpadding_tokens": 7326, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2228387320000138, "nonpadding_tokens": 1158, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21924062200002936, "nonpadding_tokens": 4934, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22183733800000027, "nonpadding_tokens": 12243, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22307776200000262, "nonpadding_tokens": 7344, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22355494900000394, "nonpadding_tokens": 5687, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003582110000479588, "prefetch": {"data_wait_seconds": 0.018027109000811947, "consumed_batches": 1912, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2462, "tokens": 181685077, "loss_tokens": 58627, "loss": 2.7593264408890104, "lr": 0.0002992983699853968, "optimizer_lrs": [0.01995322466569312, 0.0002992983699853968], "gradient_norm": 0.4206503927707672, "seconds": 0.3132109240000318, "nonpadding_tokens_per_second": 187180.57228423504, "ranks": [{"compute_seconds": 0.30960726800003613, "nonpadding_tokens": 4198, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31048925999994026, "nonpadding_tokens": 3118, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30949251399999866, "nonpadding_tokens": 2408, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30840967500000716, "nonpadding_tokens": 5421, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31090540599996075, "nonpadding_tokens": 5820, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30931101999999555, "nonpadding_tokens": 3654, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31023672499998156, "nonpadding_tokens": 19225, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30631971500008603, "nonpadding_tokens": 14783, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003363670000453567, "prefetch": {"data_wait_seconds": 0.018035793000720446, "consumed_batches": 1913, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2463, "tokens": 181728069, "loss_tokens": 42992, "loss": 2.3238358662658167, "lr": 0.000299297997133243, "optimizer_lrs": [0.019953199808882868, 0.000299297997133243], "gradient_norm": 0.4140748083591461, "seconds": 0.2661772069999415, "nonpadding_tokens_per_second": 161516.45922112875, "ranks": [{"compute_seconds": 0.2634985660000666, "nonpadding_tokens": 5471, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26327512799991837, "nonpadding_tokens": 6410, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2611703510000325, "nonpadding_tokens": 7126, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26141291300007197, "nonpadding_tokens": 4729, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2638504969999076, "nonpadding_tokens": 5949, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26140351699996245, "nonpadding_tokens": 5907, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26001403699990533, "nonpadding_tokens": 3694, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2602050579999968, "nonpadding_tokens": 3706, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003840419999505684, "prefetch": {"data_wait_seconds": 0.018044923000687163, "consumed_batches": 1914, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2464, "tokens": 181821939, "loss_tokens": 93870, "loss": 2.2736876797698944, "lr": 0.00029929718269354797, "optimizer_lrs": [0.0199531455129032, 0.00029929718269354797], "gradient_norm": 0.3727560043334961, "seconds": 0.26580015900003673, "nonpadding_tokens_per_second": 353160.05962203746, "ranks": [{"compute_seconds": 0.2632859799999778, "nonpadding_tokens": 2429, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2605524589999959, "nonpadding_tokens": 16592, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2634635660000413, "nonpadding_tokens": 23193, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26202367700000195, "nonpadding_tokens": 12307, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2600078770000209, "nonpadding_tokens": 18051, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2614776970000321, "nonpadding_tokens": 7562, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2596798360000321, "nonpadding_tokens": 7256, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26099880600008873, "nonpadding_tokens": 6480, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003441799999563955, "prefetch": {"data_wait_seconds": 0.01805270400070924, "consumed_batches": 1915, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2465, "tokens": 181875336, "loss_tokens": 53397, "loss": 2.271514551379291, "lr": 0.00029929671919762946, "optimizer_lrs": [0.0199531146131753, 0.00029929671919762946], "gradient_norm": 0.3801206946372986, "seconds": 0.30352743499997814, "nonpadding_tokens_per_second": 175921.49454300184, "ranks": [{"compute_seconds": 0.3000160459999961, "nonpadding_tokens": 16086, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.300830575999953, "nonpadding_tokens": 12919, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30003197999997155, "nonpadding_tokens": 4449, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.300373411999999, "nonpadding_tokens": 5563, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29632852800000364, "nonpadding_tokens": 2874, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2998519819999501, "nonpadding_tokens": 4380, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3008764560000827, "nonpadding_tokens": 3120, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2969569050000018, "nonpadding_tokens": 4006, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039764699999977893, "prefetch": {"data_wait_seconds": 0.01806122100067853, "consumed_batches": 1916, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2466, "tokens": 181967336, "loss_tokens": 92000, "loss": 2.4936547214673914, "lr": 0.00029929592026301807, "optimizer_lrs": [0.019953061350867874, 0.00029929592026301807], "gradient_norm": 0.5075862407684326, "seconds": 0.3611673839999412, "nonpadding_tokens_per_second": 254729.53559952407, "ranks": [{"compute_seconds": 0.35748717000001307, "nonpadding_tokens": 17779, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35789775399996415, "nonpadding_tokens": 27942, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35730656099997304, "nonpadding_tokens": 17247, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3558489439999448, "nonpadding_tokens": 5533, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3542726030000267, "nonpadding_tokens": 2484, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3568906190000689, "nonpadding_tokens": 1483, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35831432400004815, "nonpadding_tokens": 10181, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3588450999999395, "nonpadding_tokens": 9351, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000334447000000182, "prefetch": {"data_wait_seconds": 0.018069114000766717, "consumed_batches": 1917, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2467, "tokens": 182027803, "loss_tokens": 60467, "loss": 2.326268873931235, "lr": 0.0002992953949168858, "optimizer_lrs": [0.01995302632779239, 0.0002992953949168858], "gradient_norm": 0.5166280269622803, "seconds": 0.2967470940000112, "nonpadding_tokens_per_second": 203766.1066362379, "ranks": [{"compute_seconds": 0.26676499999996395, "nonpadding_tokens": 3645, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27995427700000164, "nonpadding_tokens": 4646, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27751553100006277, "nonpadding_tokens": 7763, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2776524700000209, "nonpadding_tokens": 3494, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27669080499993015, "nonpadding_tokens": 2602, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2778041049999729, "nonpadding_tokens": 17843, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28049602800001594, "nonpadding_tokens": 18371, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27701247400000284, "nonpadding_tokens": 2103, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0022531470000330955, "prefetch": {"data_wait_seconds": 0.018077457000757136, "consumed_batches": 1918, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2468, "tokens": 182084414, "loss_tokens": 56611, "loss": 2.3786551531504476, "lr": 0.0002992949028951328, "optimizer_lrs": [0.01995299352634219, 0.0002992949028951328], "gradient_norm": 0.4560902416706085, "seconds": 0.41459074700003384, "nonpadding_tokens_per_second": 136546.70397165275, "ranks": [{"compute_seconds": 0.41173349300004247, "nonpadding_tokens": 8410, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.41062235900005817, "nonpadding_tokens": 3286, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.407969855000033, "nonpadding_tokens": 5653, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.411407392000001, "nonpadding_tokens": 2916, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4068478789999972, "nonpadding_tokens": 7491, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4087584369999604, "nonpadding_tokens": 11887, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.40682713199998943, "nonpadding_tokens": 9546, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4120051129999638, "nonpadding_tokens": 7422, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003849010000749331, "prefetch": {"data_wait_seconds": 0.0180860760007846, "consumed_batches": 1919, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2469, "tokens": 182136226, "loss_tokens": 51812, "loss": 2.256135010953061, "lr": 0.0002992944524326928, "optimizer_lrs": [0.019952963495512854, 0.0002992944524326928], "gradient_norm": 0.42487797141075134, "seconds": 0.26596922999999606, "nonpadding_tokens_per_second": 194804.48922606863, "ranks": [{"compute_seconds": 0.26249797600007696, "nonpadding_tokens": 6814, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2627181540000265, "nonpadding_tokens": 2780, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26232951599990884, "nonpadding_tokens": 8128, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2607236879999846, "nonpadding_tokens": 4833, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25935172300000886, "nonpadding_tokens": 9429, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26059870200003843, "nonpadding_tokens": 11851, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25960969200002637, "nonpadding_tokens": 3527, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26356666599997425, "nonpadding_tokens": 4450, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004356770000413235, "prefetch": {"data_wait_seconds": 0.01809613600080695, "consumed_batches": 1920, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2470, "tokens": 182199281, "loss_tokens": 63055, "loss": 2.20545481127587, "lr": 0.00029929390402813106, "optimizer_lrs": [0.01995292693520874, 0.00029929390402813106], "gradient_norm": 0.47141677141189575, "seconds": 0.2510922419999133, "nonpadding_tokens_per_second": 251122.85229434437, "ranks": [{"compute_seconds": 0.24589964799997688, "nonpadding_tokens": 4529, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2445546540000123, "nonpadding_tokens": 1929, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24515990199995485, "nonpadding_tokens": 12337, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24571536399992056, "nonpadding_tokens": 6896, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24822820700001103, "nonpadding_tokens": 15742, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24510204399996383, "nonpadding_tokens": 7707, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24775721299999987, "nonpadding_tokens": 9890, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2486967069999082, "nonpadding_tokens": 4025, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034568599994599936, "prefetch": {"data_wait_seconds": 0.018104980000842374, "consumed_batches": 1921, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2471, "tokens": 182264892, "loss_tokens": 65611, "loss": 2.538355325326546, "lr": 0.0002992933331678363, "optimizer_lrs": [0.019952888877855755, 0.0002992933331678363], "gradient_norm": 0.47677382826805115, "seconds": 0.25777685200000633, "nonpadding_tokens_per_second": 254526.34513512638, "ranks": [{"compute_seconds": 0.2549114380000219, "nonpadding_tokens": 6669, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25469919899990145, "nonpadding_tokens": 6169, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2519955459999892, "nonpadding_tokens": 8760, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2553976579999926, "nonpadding_tokens": 4278, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2553490849999207, "nonpadding_tokens": 8614, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2535445299999992, "nonpadding_tokens": 2800, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.250714773000027, "nonpadding_tokens": 18366, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25183288300002005, "nonpadding_tokens": 9955, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033605199996600277, "prefetch": {"data_wait_seconds": 0.01811343600077464, "consumed_batches": 1922, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2472, "tokens": 182328119, "loss_tokens": 63227, "loss": 1.7225624041153305, "lr": 0.0002992927828322852, "optimizer_lrs": [0.019952852188819016, 0.0002992927828322852], "gradient_norm": 0.68571537733078, "seconds": 0.30466775899992626, "nonpadding_tokens_per_second": 207527.70233234722, "ranks": [{"compute_seconds": 0.30148125799996706, "nonpadding_tokens": 14199, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29797293399997216, "nonpadding_tokens": 6706, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29835494899998594, "nonpadding_tokens": 1811, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30123544299999594, "nonpadding_tokens": 3165, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30130089800002224, "nonpadding_tokens": 2331, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2990573999999242, "nonpadding_tokens": 9954, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2972372980000273, "nonpadding_tokens": 12312, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3022687599999472, "nonpadding_tokens": 12749, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040261300000565825, "prefetch": {"data_wait_seconds": 0.018124091000757403, "consumed_batches": 1923, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2473, "tokens": 182457903, "loss_tokens": 129784, "loss": 1.5011947736238673, "lr": 0.00029929165250708507, "optimizer_lrs": [0.019952776833805672, 0.00029929165250708507], "gradient_norm": 1.4736247062683105, "seconds": 0.2830121299999746, "nonpadding_tokens_per_second": 458581.0509253142, "ranks": [{"compute_seconds": 0.2778521919999548, "nonpadding_tokens": 8726, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2762835730000006, "nonpadding_tokens": 12439, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2789665079999395, "nonpadding_tokens": 28866, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27953989399998136, "nonpadding_tokens": 23912, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2802237260000311, "nonpadding_tokens": 13062, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27885435700000016, "nonpadding_tokens": 24997, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.276092196000036, "nonpadding_tokens": 12031, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2805592260000367, "nonpadding_tokens": 5751, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040446600007726374, "prefetch": {"data_wait_seconds": 0.01813370200075326, "consumed_batches": 1924, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2474, "tokens": 182520730, "loss_tokens": 62827, "loss": 2.5273628376335013, "lr": 0.00029929110500578687, "optimizer_lrs": [0.019952740333719125, 0.00029929110500578687], "gradient_norm": 0.523805558681488, "seconds": 0.3212389010000152, "nonpadding_tokens_per_second": 195577.18509314983, "ranks": [{"compute_seconds": 0.3180547399999796, "nonpadding_tokens": 9713, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3188736910000216, "nonpadding_tokens": 3545, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3179544730000998, "nonpadding_tokens": 4144, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31647452600009274, "nonpadding_tokens": 18556, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31547687599993424, "nonpadding_tokens": 6643, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31576274700000795, "nonpadding_tokens": 2555, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3152059220000183, "nonpadding_tokens": 9465, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3187876340000457, "nonpadding_tokens": 8206, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004002889999128456, "prefetch": {"data_wait_seconds": 0.01814360400074122, "consumed_batches": 1925, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2475, "tokens": 182564955, "loss_tokens": 44225, "loss": 2.271825890333522, "lr": 0.0002992907194837488, "optimizer_lrs": [0.019952714632249922, 0.0002992907194837488], "gradient_norm": 0.47787174582481384, "seconds": 0.3432483139999931, "nonpadding_tokens_per_second": 128842.5847883433, "ranks": [{"compute_seconds": 0.34073844799991093, "nonpadding_tokens": 13912, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3399771800000053, "nonpadding_tokens": 7115, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3374092499999506, "nonpadding_tokens": 2656, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3398986509999986, "nonpadding_tokens": 3300, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34051844799989794, "nonpadding_tokens": 4728, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3389478109999118, "nonpadding_tokens": 1855, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33669248199998947, "nonpadding_tokens": 5418, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3369686759999695, "nonpadding_tokens": 5241, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003625470000088171, "prefetch": {"data_wait_seconds": 0.01815323400076352, "consumed_batches": 1926, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2476, "tokens": 182604564, "loss_tokens": 39609, "loss": 2.482477348897978, "lr": 0.00029929037411197953, "optimizer_lrs": [0.019952691607465302, 0.00029929037411197953], "gradient_norm": 0.47548383474349976, "seconds": 0.21785027900000387, "nonpadding_tokens_per_second": 181817.5316635665, "ranks": [{"compute_seconds": 0.2128919889998997, "nonpadding_tokens": 3059, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21199916600005508, "nonpadding_tokens": 3467, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20954580400007217, "nonpadding_tokens": 2984, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20980797200002144, "nonpadding_tokens": 6104, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21445860900007574, "nonpadding_tokens": 4342, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21114215400007197, "nonpadding_tokens": 4581, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21420009700000264, "nonpadding_tokens": 3414, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21523757099998875, "nonpadding_tokens": 11658, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004122679999909451, "prefetch": {"data_wait_seconds": 0.018162036000788362, "consumed_batches": 1927, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2477, "tokens": 182645509, "loss_tokens": 40945, "loss": 1.9762314537794603, "lr": 0.0002992900170027872, "optimizer_lrs": [0.019952667800185817, 0.0002992900170027872], "gradient_norm": 0.42834532260894775, "seconds": 0.29959624499997517, "nonpadding_tokens_per_second": 136667.26697460242, "ranks": [{"compute_seconds": 0.2966396660000328, "nonpadding_tokens": 2635, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29295724400003564, "nonpadding_tokens": 6103, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2960053089999519, "nonpadding_tokens": 3956, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29615328499994575, "nonpadding_tokens": 4395, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29697214499992697, "nonpadding_tokens": 2353, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2956475800000362, "nonpadding_tokens": 3616, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2927583489999961, "nonpadding_tokens": 3912, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29724500099996476, "nonpadding_tokens": 13975, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000416604999941228, "prefetch": {"data_wait_seconds": 0.01818649200079392, "consumed_batches": 1928, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2478, "tokens": 182713660, "loss_tokens": 68151, "loss": 2.356416037548972, "lr": 0.00029928942241289427, "optimizer_lrs": [0.01995262816085962, 0.00029928942241289427], "gradient_norm": 0.5607715249061584, "seconds": 0.28550843599998643, "nonpadding_tokens_per_second": 238700.4774878289, "ranks": [{"compute_seconds": 0.2808077229999526, "nonpadding_tokens": 4826, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27976905800005625, "nonpadding_tokens": 11112, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2803025940000907, "nonpadding_tokens": 7379, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2831806660000211, "nonpadding_tokens": 8602, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2790116300000136, "nonpadding_tokens": 4382, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28170990099999926, "nonpadding_tokens": 5255, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28303511900003286, "nonpadding_tokens": 6761, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27961905600000136, "nonpadding_tokens": 19834, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003532120000500072, "prefetch": {"data_wait_seconds": 0.018195910000784465, "consumed_batches": 1929, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2479, "tokens": 182791196, "loss_tokens": 77536, "loss": 2.2615238808424474, "lr": 0.00029928874564083443, "optimizer_lrs": [0.019952583042722297, 0.00029928874564083443], "gradient_norm": 0.9311239719390869, "seconds": 0.26775572400003966, "nonpadding_tokens_per_second": 289577.3761310459, "ranks": [{"compute_seconds": 0.26351422100003674, "nonpadding_tokens": 10929, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2642015830000446, "nonpadding_tokens": 8800, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26353606499992566, "nonpadding_tokens": 3277, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26397998000004463, "nonpadding_tokens": 7856, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2644844589999593, "nonpadding_tokens": 5440, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26271557899997333, "nonpadding_tokens": 20001, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26408230300000923, "nonpadding_tokens": 18419, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26539500400008365, "nonpadding_tokens": 2814, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004206280000289553, "prefetch": {"data_wait_seconds": 0.01820387800076162, "consumed_batches": 1930, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2480, "tokens": 182842019, "loss_tokens": 50823, "loss": 2.250624255996301, "lr": 0.0002992883018585633, "optimizer_lrs": [0.019952553457237555, 0.0002992883018585633], "gradient_norm": 0.4723254442214966, "seconds": 0.2758479809999699, "nonpadding_tokens_per_second": 184242.78407172952, "ranks": [{"compute_seconds": 0.2716809360000525, "nonpadding_tokens": 8185, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2695085809999682, "nonpadding_tokens": 9398, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2722637429999395, "nonpadding_tokens": 3686, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27283672600003683, "nonpadding_tokens": 2654, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2692640879999999, "nonpadding_tokens": 12206, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27183124500004396, "nonpadding_tokens": 7855, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27294045799999367, "nonpadding_tokens": 1912, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27354917700006354, "nonpadding_tokens": 4927, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033508500007428665, "prefetch": {"data_wait_seconds": 0.018212108000696503, "consumed_batches": 1931, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2481, "tokens": 182935922, "loss_tokens": 93903, "loss": 2.6155034982907894, "lr": 0.00029928748154212196, "optimizer_lrs": [0.0199524987694748, 0.00029928748154212196], "gradient_norm": 0.6538229584693909, "seconds": 0.3468333600000051, "nonpadding_tokens_per_second": 270743.85232146824, "ranks": [{"compute_seconds": 0.3412926810000272, "nonpadding_tokens": 17693, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3427848730000278, "nonpadding_tokens": 18535, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33937535299992305, "nonpadding_tokens": 7933, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34440643200002796, "nonpadding_tokens": 9970, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33881158700000924, "nonpadding_tokens": 10581, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3416210020000108, "nonpadding_tokens": 13418, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33876161600005616, "nonpadding_tokens": 10449, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3395592189999661, "nonpadding_tokens": 5324, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040321599999515456, "prefetch": {"data_wait_seconds": 0.018221816000689728, "consumed_batches": 1932, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2482, "tokens": 183002135, "loss_tokens": 66213, "loss": 2.398227359053358, "lr": 0.0002992869028362881, "optimizer_lrs": [0.019952460189085874, 0.0002992869028362881], "gradient_norm": 0.4778556227684021, "seconds": 0.2790691689999676, "nonpadding_tokens_per_second": 237263.7587923863, "ranks": [{"compute_seconds": 0.27664113299999826, "nonpadding_tokens": 5451, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2753255820000504, "nonpadding_tokens": 10245, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2759673019999127, "nonpadding_tokens": 3569, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2764009879999776, "nonpadding_tokens": 7578, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2709761119999712, "nonpadding_tokens": 8000, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2745941880000373, "nonpadding_tokens": 4764, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27012887099999716, "nonpadding_tokens": 12913, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2713653699998986, "nonpadding_tokens": 13693, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004101979999404648, "prefetch": {"data_wait_seconds": 0.018229836000685395, "consumed_batches": 1933, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2483, "tokens": 183049768, "loss_tokens": 47633, "loss": 2.1925842837423635, "lr": 0.0002992864863759359, "optimizer_lrs": [0.019952432425062394, 0.0002992864863759359], "gradient_norm": 0.43203145265579224, "seconds": 0.24262026600001718, "nonpadding_tokens_per_second": 196327.37522428003, "ranks": [{"compute_seconds": 0.24021583599994756, "nonpadding_tokens": 3573, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23654475599994385, "nonpadding_tokens": 7007, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.239310506000038, "nonpadding_tokens": 2365, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24003725600005055, "nonpadding_tokens": 3978, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23612971700003982, "nonpadding_tokens": 3823, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23879430199997387, "nonpadding_tokens": 12336, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23555765699995845, "nonpadding_tokens": 8284, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2364183629999843, "nonpadding_tokens": 6267, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038414699997701973, "prefetch": {"data_wait_seconds": 0.018238746000704964, "consumed_batches": 1934, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2484, "tokens": 183124110, "loss_tokens": 74342, "loss": 2.615765650641629, "lr": 0.00029928583615367743, "optimizer_lrs": [0.01995238907691183, 0.00029928583615367743], "gradient_norm": 0.4508246183395386, "seconds": 0.32069213500005844, "nonpadding_tokens_per_second": 231817.3471887187, "ranks": [{"compute_seconds": 0.3145006409999951, "nonpadding_tokens": 15628, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31411192899997786, "nonpadding_tokens": 6081, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3137743220000857, "nonpadding_tokens": 6914, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3144696329999306, "nonpadding_tokens": 8016, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31294933399999536, "nonpadding_tokens": 9067, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3145383879999599, "nonpadding_tokens": 4280, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31299283800001376, "nonpadding_tokens": 4033, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31827410599998984, "nonpadding_tokens": 20323, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003573709999500352, "prefetch": {"data_wait_seconds": 0.018248261000692878, "consumed_batches": 1935, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2485, "tokens": 183172934, "loss_tokens": 48824, "loss": 2.2076749843826806, "lr": 0.0002992854089604767, "optimizer_lrs": [0.019952360597365115, 0.0002992854089604767], "gradient_norm": 0.6135966181755066, "seconds": 0.263977342999965, "nonpadding_tokens_per_second": 184955.26716475235, "ranks": [{"compute_seconds": 0.2608106229999976, "nonpadding_tokens": 5111, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2615970820000939, "nonpadding_tokens": 3007, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26042051599995375, "nonpadding_tokens": 5845, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26101642000003267, "nonpadding_tokens": 1786, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25749092700004894, "nonpadding_tokens": 3115, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2604276189999837, "nonpadding_tokens": 10502, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2614572960000032, "nonpadding_tokens": 7532, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25824642899999617, "nonpadding_tokens": 11926, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036265200003526843, "prefetch": {"data_wait_seconds": 0.01825716200069394, "consumed_batches": 1936, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2486, "tokens": 183249963, "loss_tokens": 77029, "loss": 2.286969233989796, "lr": 0.0002992847347241963, "optimizer_lrs": [0.019952315648279755, 0.0002992847347241963], "gradient_norm": 0.45549672842025757, "seconds": 0.2592689489999884, "nonpadding_tokens_per_second": 297100.7530871097, "ranks": [{"compute_seconds": 0.25582897300000695, "nonpadding_tokens": 3401, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2562967270000627, "nonpadding_tokens": 7464, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2554679549999719, "nonpadding_tokens": 1765, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25395050500003435, "nonpadding_tokens": 4383, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.256878188000087, "nonpadding_tokens": 14364, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25422019500001625, "nonpadding_tokens": 19714, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25216176899994025, "nonpadding_tokens": 5762, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2529676719999543, "nonpadding_tokens": 20176, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000366313999961676, "prefetch": {"data_wait_seconds": 0.01826657900069506, "consumed_batches": 1937, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2487, "tokens": 183324117, "loss_tokens": 74154, "loss": 2.361335025757208, "lr": 0.00029928408535329065, "optimizer_lrs": [0.019952272356886045, 0.00029928408535329065], "gradient_norm": 0.4101123511791229, "seconds": 0.4077984360000073, "nonpadding_tokens_per_second": 181839.83422633496, "ranks": [{"compute_seconds": 0.40534866500001954, "nonpadding_tokens": 12819, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.40209201199991185, "nonpadding_tokens": 3393, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.40234683699998186, "nonpadding_tokens": 10836, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.405324553000014, "nonpadding_tokens": 6423, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.40106288899994524, "nonpadding_tokens": 14548, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.404011942000011, "nonpadding_tokens": 19127, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.40123448400004236, "nonpadding_tokens": 4319, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4023689870000453, "nonpadding_tokens": 2689, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004273780000403349, "prefetch": {"data_wait_seconds": 0.01827780300068582, "consumed_batches": 1938, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2488, "tokens": 183392146, "loss_tokens": 68029, "loss": 2.61330898403622, "lr": 0.00029928348936091693, "optimizer_lrs": [0.01995223262406113, 0.00029928348936091693], "gradient_norm": 0.42819446325302124, "seconds": 0.2649089049999702, "nonpadding_tokens_per_second": 256801.4842687438, "ranks": [{"compute_seconds": 0.2625244500000008, "nonpadding_tokens": 9652, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26183071900004506, "nonpadding_tokens": 10132, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.260980399999994, "nonpadding_tokens": 11228, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2595217389999789, "nonpadding_tokens": 15017, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25775053100005607, "nonpadding_tokens": 6297, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2592702440000494, "nonpadding_tokens": 5724, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26204298299990114, "nonpadding_tokens": 6519, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26253861299994696, "nonpadding_tokens": 3460, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003956279999783874, "prefetch": {"data_wait_seconds": 0.0182876120007478, "consumed_batches": 1939, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2489, "tokens": 183451531, "loss_tokens": 59385, "loss": 2.115097588827145, "lr": 0.00029928296889522024, "optimizer_lrs": [0.019952197926348017, 0.00029928296889522024], "gradient_norm": 0.44901853799819946, "seconds": 0.24785461599992686, "nonpadding_tokens_per_second": 239596.1025798185, "ranks": [{"compute_seconds": 0.24381951499992738, "nonpadding_tokens": 2613, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24559374099999332, "nonpadding_tokens": 8406, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2424432389999538, "nonpadding_tokens": 5857, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24551193100001, "nonpadding_tokens": 4368, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24196036099999674, "nonpadding_tokens": 4735, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24340181200000188, "nonpadding_tokens": 5855, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24191939399997864, "nonpadding_tokens": 3521, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24245900600010373, "nonpadding_tokens": 24030, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039438999999674706, "prefetch": {"data_wait_seconds": 0.018296481000746212, "consumed_batches": 1940, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2490, "tokens": 183504940, "loss_tokens": 53409, "loss": 2.4190022690464152, "lr": 0.0002992825006437876, "optimizer_lrs": [0.019952166709585842, 0.0002992825006437876], "gradient_norm": 0.3936936855316162, "seconds": 0.2663055110000414, "nonpadding_tokens_per_second": 200555.36890482035, "ranks": [{"compute_seconds": 0.26317783900003633, "nonpadding_tokens": 11224, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26270532599994567, "nonpadding_tokens": 11259, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2629444839999451, "nonpadding_tokens": 6277, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2610488779999969, "nonpadding_tokens": 2400, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26350243499996395, "nonpadding_tokens": 2298, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2621035470000379, "nonpadding_tokens": 13223, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25927348100003655, "nonpadding_tokens": 2296, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2639901310000141, "nonpadding_tokens": 4432, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034039900003790535, "prefetch": {"data_wait_seconds": 0.01830472000081329, "consumed_batches": 1941, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2491, "tokens": 183559618, "loss_tokens": 54678, "loss": 2.3946588435934015, "lr": 0.00029928202110876946, "optimizer_lrs": [0.01995213474058463, 0.00029928202110876946], "gradient_norm": 0.40921303629875183, "seconds": 0.30969668900002034, "nonpadding_tokens_per_second": 176553.38898374987, "ranks": [{"compute_seconds": 0.28929755199999363, "nonpadding_tokens": 7407, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2919992670000511, "nonpadding_tokens": 11662, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29131997399997545, "nonpadding_tokens": 2358, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2916849560000401, "nonpadding_tokens": 4498, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2876883359999738, "nonpadding_tokens": 7495, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.289270045999956, "nonpadding_tokens": 4717, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2915876949999756, "nonpadding_tokens": 9991, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28867034799998237, "nonpadding_tokens": 6550, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.013723381999966477, "prefetch": {"data_wait_seconds": 0.0183150050008436, "consumed_batches": 1942, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2492, "tokens": 183616304, "loss_tokens": 56686, "loss": 2.3202510320008467, "lr": 0.0002992815237945878, "optimizer_lrs": [0.019952101586305854, 0.0002992815237945878], "gradient_norm": 0.4315762221813202, "seconds": 0.22735260000001745, "nonpadding_tokens_per_second": 249330.77519234727, "ranks": [{"compute_seconds": 0.2250774699999738, "nonpadding_tokens": 8284, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22030571499999496, "nonpadding_tokens": 1513, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22208853800009365, "nonpadding_tokens": 3084, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2228798830000187, "nonpadding_tokens": 4293, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22456809000004796, "nonpadding_tokens": 1867, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2240843249999216, "nonpadding_tokens": 17995, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22155531999999312, "nonpadding_tokens": 4247, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22457824600007825, "nonpadding_tokens": 15403, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003442930000119304, "prefetch": {"data_wait_seconds": 0.01832337600076528, "consumed_batches": 1943, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2493, "tokens": 183666819, "loss_tokens": 50515, "loss": 2.229795759922795, "lr": 0.0002992810804747569, "optimizer_lrs": [0.01995207203165046, 0.0002992810804747569], "gradient_norm": 0.37973523139953613, "seconds": 0.22625687300001118, "nonpadding_tokens_per_second": 223263.93594239015, "ranks": [{"compute_seconds": 0.22210195499997099, "nonpadding_tokens": 2612, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21997607399998742, "nonpadding_tokens": 2297, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22287976400002663, "nonpadding_tokens": 12307, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2209865859999809, "nonpadding_tokens": 3849, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22391490600000452, "nonpadding_tokens": 5735, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2207774219999692, "nonpadding_tokens": 3438, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21949437199998556, "nonpadding_tokens": 11396, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22029815400003372, "nonpadding_tokens": 8881, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003817549999212133, "prefetch": {"data_wait_seconds": 0.018332827000747898, "consumed_batches": 1944, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2494, "tokens": 183729588, "loss_tokens": 62769, "loss": 2.5192456666507352, "lr": 0.0002992805294237818, "optimizer_lrs": [0.019952035294918787, 0.0002992805294237818], "gradient_norm": 0.6395700573921204, "seconds": 0.2827170360000082, "nonpadding_tokens_per_second": 222020.57890843967, "ranks": [{"compute_seconds": 0.27961201299990535, "nonpadding_tokens": 5905, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2803848779999498, "nonpadding_tokens": 9545, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27890273800005616, "nonpadding_tokens": 5086, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2773389289999386, "nonpadding_tokens": 3053, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28033501299989894, "nonpadding_tokens": 10743, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2784064989999706, "nonpadding_tokens": 10516, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2798313800000187, "nonpadding_tokens": 12908, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2763994189999721, "nonpadding_tokens": 5013, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038619800000105897, "prefetch": {"data_wait_seconds": 0.018341776000809773, "consumed_batches": 1945, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2495, "tokens": 183829704, "loss_tokens": 100116, "loss": 2.3239166004934275, "lr": 0.0002992796500666281, "optimizer_lrs": [0.01995197667110854, 0.0002992796500666281], "gradient_norm": 0.5369148850440979, "seconds": 0.29483705500001633, "nonpadding_tokens_per_second": 339563.83128299273, "ranks": [{"compute_seconds": 0.28789473500000895, "nonpadding_tokens": 3054, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2900086380000175, "nonpadding_tokens": 3954, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2869985940000106, "nonpadding_tokens": 19567, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29171424599996953, "nonpadding_tokens": 18117, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2920953799999779, "nonpadding_tokens": 16190, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.288644899000019, "nonpadding_tokens": 18367, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2856154800000468, "nonpadding_tokens": 15192, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29242855000006784, "nonpadding_tokens": 5675, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038724500006992457, "prefetch": {"data_wait_seconds": 0.018350103000742024, "consumed_batches": 1946, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2496, "tokens": 183887002, "loss_tokens": 57298, "loss": 2.4642161375615204, "lr": 0.00029927914655536664, "optimizer_lrs": [0.01995194310369111, 0.00029927914655536664], "gradient_norm": 0.6347938776016235, "seconds": 0.2915793360000407, "nonpadding_tokens_per_second": 196509.12436398442, "ranks": [{"compute_seconds": 0.28909953400000177, "nonpadding_tokens": 2199, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28823491200000717, "nonpadding_tokens": 7722, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28558140400002685, "nonpadding_tokens": 2952, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2861900970000306, "nonpadding_tokens": 10763, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2847101009999733, "nonpadding_tokens": 8171, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28721486199992796, "nonpadding_tokens": 9593, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2883688639999491, "nonpadding_tokens": 10307, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2851494899999807, "nonpadding_tokens": 5591, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004440550000026633, "prefetch": {"data_wait_seconds": 0.018359722000766965, "consumed_batches": 1947, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2497, "tokens": 183920281, "loss_tokens": 33279, "loss": 2.4973718880525255, "lr": 0.0002992788540326605, "optimizer_lrs": [0.019951923602177367, 0.0002992788540326605], "gradient_norm": 0.48902612924575806, "seconds": 0.2355421419999857, "nonpadding_tokens_per_second": 141286.81906952354, "ranks": [{"compute_seconds": 0.2265876629999184, "nonpadding_tokens": 5371, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23276460699992185, "nonpadding_tokens": 1507, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2298823530001073, "nonpadding_tokens": 5539, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23040173899994443, "nonpadding_tokens": 2166, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.233138899000096, "nonpadding_tokens": 5952, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2314034429999765, "nonpadding_tokens": 3212, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23282057799997347, "nonpadding_tokens": 2359, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22928342700004123, "nonpadding_tokens": 7173, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004167079999888301, "prefetch": {"data_wait_seconds": 0.018377225000861586, "consumed_batches": 1948, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2498, "tokens": 183986950, "loss_tokens": 66669, "loss": 2.260499632512862, "lr": 0.00029927826783351783, "optimizer_lrs": [0.019951884522234523, 0.00029927826783351783], "gradient_norm": 0.47037020325660706, "seconds": 0.2899785189999875, "nonpadding_tokens_per_second": 229910.13344682567, "ranks": [{"compute_seconds": 0.2848555829999668, "nonpadding_tokens": 18534, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28670792499997333, "nonpadding_tokens": 14861, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2858818459999384, "nonpadding_tokens": 3017, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2840679189999946, "nonpadding_tokens": 4260, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28277784700003394, "nonpadding_tokens": 3124, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2855441530000462, "nonpadding_tokens": 2623, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.282449693999979, "nonpadding_tokens": 6860, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28758279399994535, "nonpadding_tokens": 13390, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042896599995856377, "prefetch": {"data_wait_seconds": 0.018387055000815167, "consumed_batches": 1949, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2499, "tokens": 184034029, "loss_tokens": 47079, "loss": 2.789257152870707, "lr": 0.0002992778537398815, "optimizer_lrs": [0.019951856915992102, 0.0002992778537398815], "gradient_norm": 0.47916996479034424, "seconds": 0.2664990590000116, "nonpadding_tokens_per_second": 176657.28418199765, "ranks": [{"compute_seconds": 0.24032811899996886, "nonpadding_tokens": 6308, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.257435399999963, "nonpadding_tokens": 9921, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2542705900000328, "nonpadding_tokens": 2720, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2547055410000212, "nonpadding_tokens": 2416, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25341700100000253, "nonpadding_tokens": 11659, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2558841530000109, "nonpadding_tokens": 7551, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25314418400000704, "nonpadding_tokens": 3568, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2552435069999319, "nonpadding_tokens": 2936, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004502859999320208, "prefetch": {"data_wait_seconds": 0.018396492000832154, "consumed_batches": 1950, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2500, "tokens": 184107656, "loss_tokens": 73627, "loss": 2.2358255123799693, "lr": 0.00029927720590005273, "optimizer_lrs": [0.019951813726670185, 0.00029927720590005273], "gradient_norm": 0.41430479288101196, "seconds": 0.278013595999937, "nonpadding_tokens_per_second": 264832.3717233479, "ranks": [{"compute_seconds": 0.2741669260000208, "nonpadding_tokens": 9802, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27498349099994357, "nonpadding_tokens": 12047, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27165501199999653, "nonpadding_tokens": 2906, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27233648100002483, "nonpadding_tokens": 6592, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27550622499995825, "nonpadding_tokens": 2626, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2732717059999459, "nonpadding_tokens": 7095, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27038129499999286, "nonpadding_tokens": 11896, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27118607099998826, "nonpadding_tokens": 20663, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003917279999541279, "prefetch": {"data_wait_seconds": 0.01840556400084381, "consumed_batches": 1951, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2501, "tokens": 184171620, "loss_tokens": 63964, "loss": 2.460409370896129, "lr": 0.00029927664284927025, "optimizer_lrs": [0.01995177618995135, 0.00029927664284927025], "gradient_norm": 0.4432515501976013, "seconds": 0.3710410350000757, "nonpadding_tokens_per_second": 172390.63598447258, "ranks": [{"compute_seconds": 0.36811990999990485, "nonpadding_tokens": 27353, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36809530700008963, "nonpadding_tokens": 4405, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3651997659999324, "nonpadding_tokens": 2686, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3678127599999925, "nonpadding_tokens": 1575, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36825859299995045, "nonpadding_tokens": 12211, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3656769980000263, "nonpadding_tokens": 3851, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36419536699997934, "nonpadding_tokens": 5325, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.36467349899999135, "nonpadding_tokens": 6558, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004342839999935677, "prefetch": {"data_wait_seconds": 0.018415200000802656, "consumed_batches": 1952, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2502, "tokens": 184217967, "loss_tokens": 46347, "loss": 2.4987450306923855, "lr": 0.00029927623473764296, "optimizer_lrs": [0.019951748982509534, 0.00029927623473764296], "gradient_norm": 0.46497151255607605, "seconds": 0.3313895000000002, "nonpadding_tokens_per_second": 139856.5736090008, "ranks": [{"compute_seconds": 0.3261926239999866, "nonpadding_tokens": 9569, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32809644999997545, "nonpadding_tokens": 3827, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3275540480000245, "nonpadding_tokens": 1844, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32509336299995084, "nonpadding_tokens": 9461, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32418183200002204, "nonpadding_tokens": 4139, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3267736080000532, "nonpadding_tokens": 8301, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3237910280000733, "nonpadding_tokens": 2797, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32882904699999926, "nonpadding_tokens": 6409, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003534670000817641, "prefetch": {"data_wait_seconds": 0.0184252470008488, "consumed_batches": 1953, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2503, "tokens": 184264194, "loss_tokens": 46227, "loss": 2.474283833041296, "lr": 0.00029927582756834463, "optimizer_lrs": [0.019951721837889645, 0.00029927582756834463], "gradient_norm": 0.45900383591651917, "seconds": 0.22959359300000415, "nonpadding_tokens_per_second": 201342.72649323958, "ranks": [{"compute_seconds": 0.2273725509999167, "nonpadding_tokens": 2792, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22697001799997452, "nonpadding_tokens": 9652, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22424832299998343, "nonpadding_tokens": 10425, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22669500300003165, "nonpadding_tokens": 3713, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2229433439999866, "nonpadding_tokens": 4117, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22609274199999163, "nonpadding_tokens": 3413, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22294719599995005, "nonpadding_tokens": 3195, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22375733799992759, "nonpadding_tokens": 8920, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003571140000531159, "prefetch": {"data_wait_seconds": 0.01843414400089216, "consumed_batches": 1954, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2504, "tokens": 184310187, "loss_tokens": 45993, "loss": 2.0813583384971626, "lr": 0.00029927542234680276, "optimizer_lrs": [0.019951694823120186, 0.00029927542234680276], "gradient_norm": 1.1843026876449585, "seconds": 0.29737089599996125, "nonpadding_tokens_per_second": 154665.43840929877, "ranks": [{"compute_seconds": 0.29375174799997694, "nonpadding_tokens": 9443, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2905472219999865, "nonpadding_tokens": 11626, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2931414020000602, "nonpadding_tokens": 1688, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2935021289999895, "nonpadding_tokens": 4902, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2947110990000965, "nonpadding_tokens": 4209, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.291190959000005, "nonpadding_tokens": 5054, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2897491850000051, "nonpadding_tokens": 4551, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2947962309999639, "nonpadding_tokens": 4520, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000393168000073274, "prefetch": {"data_wait_seconds": 0.01844527700093579, "consumed_batches": 1955, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2505, "tokens": 184397421, "loss_tokens": 87234, "loss": 2.2344708270284523, "lr": 0.0002992746534607204, "optimizer_lrs": [0.01995164356404803, 0.0002992746534607204], "gradient_norm": 1.3098294734954834, "seconds": 0.31669845399994756, "nonpadding_tokens_per_second": 275448.13969952136, "ranks": [{"compute_seconds": 0.3116247460000068, "nonpadding_tokens": 2002, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.310462126999937, "nonpadding_tokens": 20184, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3103840590000573, "nonpadding_tokens": 29592, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3111894730000131, "nonpadding_tokens": 10383, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3096687040000461, "nonpadding_tokens": 9154, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.312579405000065, "nonpadding_tokens": 3814, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30960665700001755, "nonpadding_tokens": 4860, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31424769800003105, "nonpadding_tokens": 7245, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039045199991960544, "prefetch": {"data_wait_seconds": 0.01845487700093429, "consumed_batches": 1956, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2506, "tokens": 184459303, "loss_tokens": 61882, "loss": 2.3539897506544714, "lr": 0.0002992741077822517, "optimizer_lrs": [0.019951607185483448, 0.0002992741077822517], "gradient_norm": 0.5788601040840149, "seconds": 0.2893812080000089, "nonpadding_tokens_per_second": 213842.49664200068, "ranks": [{"compute_seconds": 0.28706457700002375, "nonpadding_tokens": 5446, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2843074150000575, "nonpadding_tokens": 5376, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28494671199996446, "nonpadding_tokens": 2850, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2861241699999937, "nonpadding_tokens": 8242, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2839668309999297, "nonpadding_tokens": 3016, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2855920850000757, "nonpadding_tokens": 17172, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28386736300001303, "nonpadding_tokens": 10546, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28462880699999005, "nonpadding_tokens": 9234, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036674299997230264, "prefetch": {"data_wait_seconds": 0.01846320000090884, "consumed_batches": 1957, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2507, "tokens": 184541854, "loss_tokens": 82551, "loss": 2.6731308524427324, "lr": 0.0002992733795249809, "optimizer_lrs": [0.01995155863499873, 0.0002992733795249809], "gradient_norm": 0.47495630383491516, "seconds": 0.26702579300001617, "nonpadding_tokens_per_second": 309149.9104732366, "ranks": [{"compute_seconds": 0.2620215279999911, "nonpadding_tokens": 17687, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2641266990000304, "nonpadding_tokens": 17662, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26445846700005404, "nonpadding_tokens": 9535, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26465192199998455, "nonpadding_tokens": 4416, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26023443199994745, "nonpadding_tokens": 7543, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26303811699995094, "nonpadding_tokens": 2853, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2599225840000372, "nonpadding_tokens": 2778, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2605181750000156, "nonpadding_tokens": 20077, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004101289999880464, "prefetch": {"data_wait_seconds": 0.018473208000912678, "consumed_batches": 1958, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2508, "tokens": 184596206, "loss_tokens": 54352, "loss": 2.5485587581873714, "lr": 0.00029927289983786834, "optimizer_lrs": [0.01995152665585789, 0.00029927289983786834], "gradient_norm": 0.5039998292922974, "seconds": 0.2615855640000291, "nonpadding_tokens_per_second": 207779.05007018644, "ranks": [{"compute_seconds": 0.25762081400000625, "nonpadding_tokens": 8154, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2593111820000331, "nonpadding_tokens": 1741, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25679542999989735, "nonpadding_tokens": 8612, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2573027409999895, "nonpadding_tokens": 1917, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25711734300000444, "nonpadding_tokens": 7209, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25441226899999947, "nonpadding_tokens": 4137, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.255273351000028, "nonpadding_tokens": 7013, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2564320379999572, "nonpadding_tokens": 15569, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036531899991132377, "prefetch": {"data_wait_seconds": 0.018481625000916324, "consumed_batches": 1959, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2509, "tokens": 184664620, "loss_tokens": 68414, "loss": 2.341978615488058, "lr": 0.00029927229582126, "optimizer_lrs": [0.019951486388084003, 0.00029927229582126], "gradient_norm": 0.42988985776901245, "seconds": 0.23929449800004932, "nonpadding_tokens_per_second": 285898.7589425725, "ranks": [{"compute_seconds": 0.23393820199999027, "nonpadding_tokens": 1990, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23625522500003626, "nonpadding_tokens": 10699, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23296653599993533, "nonpadding_tokens": 12765, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23687121400007527, "nonpadding_tokens": 7860, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23212544099999377, "nonpadding_tokens": 3204, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2343171820000407, "nonpadding_tokens": 13792, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23660931799997797, "nonpadding_tokens": 4589, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23266450400001304, "nonpadding_tokens": 13515, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042044799999985116, "prefetch": {"data_wait_seconds": 0.018491149000922746, "consumed_batches": 1960, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2510, "tokens": 184744507, "loss_tokens": 79887, "loss": 2.4617700235958293, "lr": 0.0002992715901947801, "optimizer_lrs": [0.019951439346318673, 0.0002992715901947801], "gradient_norm": 0.39315134286880493, "seconds": 0.2885544770000479, "nonpadding_tokens_per_second": 276852.40177364065, "ranks": [{"compute_seconds": 0.2837734839999939, "nonpadding_tokens": 15633, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2818944890000239, "nonpadding_tokens": 8727, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28263922399992225, "nonpadding_tokens": 6564, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2832941579999897, "nonpadding_tokens": 5601, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2858273670000244, "nonpadding_tokens": 4208, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.284483516000023, "nonpadding_tokens": 16124, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28165425600002436, "nonpadding_tokens": 8580, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28626908100000037, "nonpadding_tokens": 14450, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033959100005631626, "prefetch": {"data_wait_seconds": 0.01849915200098451, "consumed_batches": 1961, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2511, "tokens": 184789833, "loss_tokens": 45326, "loss": 2.249604083473062, "lr": 0.00029927118968732204, "optimizer_lrs": [0.01995141264582147, 0.00029927118968732204], "gradient_norm": 0.5177580714225769, "seconds": 0.2540896950000615, "nonpadding_tokens_per_second": 178385.82552507304, "ranks": [{"compute_seconds": 0.2493909340000755, "nonpadding_tokens": 6548, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2516118360000519, "nonpadding_tokens": 6388, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25045387299996946, "nonpadding_tokens": 2803, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24852118299997983, "nonpadding_tokens": 11738, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24714767499995105, "nonpadding_tokens": 5596, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2500265969999873, "nonpadding_tokens": 4195, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25114129700000376, "nonpadding_tokens": 4514, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25183689899995443, "nonpadding_tokens": 3544, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036796400002003793, "prefetch": {"data_wait_seconds": 0.018507413001046302, "consumed_batches": 1962, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2512, "tokens": 184846499, "loss_tokens": 56666, "loss": 2.236367762414852, "lr": 0.0002992706888234797, "optimizer_lrs": [0.01995137925489865, 0.0002992706888234797], "gradient_norm": 0.45160698890686035, "seconds": 0.2751634669999703, "nonpadding_tokens_per_second": 205935.76835549218, "ranks": [{"compute_seconds": 0.2718006609999293, "nonpadding_tokens": 13657, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2725741619999553, "nonpadding_tokens": 2876, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.272096556000065, "nonpadding_tokens": 9994, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2723001379999914, "nonpadding_tokens": 8973, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26826170100002855, "nonpadding_tokens": 9593, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2715116620000799, "nonpadding_tokens": 6420, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26866621399994983, "nonpadding_tokens": 2620, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2728956500000095, "nonpadding_tokens": 2533, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003295200000366094, "prefetch": {"data_wait_seconds": 0.018515211001002285, "consumed_batches": 1963, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2513, "tokens": 184887763, "loss_tokens": 41264, "loss": 2.0800319285575806, "lr": 0.00029927032398806714, "optimizer_lrs": [0.019951354932537812, 0.00029927032398806714], "gradient_norm": 0.435027539730072, "seconds": 0.2514388820000022, "nonpadding_tokens_per_second": 164111.4519432187, "ranks": [{"compute_seconds": 0.2484775760000275, "nonpadding_tokens": 4586, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24590081500002725, "nonpadding_tokens": 1917, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2480519959999583, "nonpadding_tokens": 2072, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24885970800005452, "nonpadding_tokens": 8898, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24430195499996898, "nonpadding_tokens": 8343, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24682734999998956, "nonpadding_tokens": 3015, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24905863499998304, "nonpadding_tokens": 5292, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24504498100009187, "nonpadding_tokens": 7141, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039497400007348915, "prefetch": {"data_wait_seconds": 0.018525040000966442, "consumed_batches": 1964, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2514, "tokens": 184941214, "loss_tokens": 53451, "loss": 2.1916651758152326, "lr": 0.00029926985126610726, "optimizer_lrs": [0.019951323417740485, 0.00029926985126610726], "gradient_norm": 0.48484140634536743, "seconds": 0.2820152470000039, "nonpadding_tokens_per_second": 189532.30567707305, "ranks": [{"compute_seconds": 0.2796708989999388, "nonpadding_tokens": 3165, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2763909950000425, "nonpadding_tokens": 10545, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27916188300002887, "nonpadding_tokens": 4370, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2772516090000181, "nonpadding_tokens": 7435, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2755894800000078, "nonpadding_tokens": 6769, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27768414199999825, "nonpadding_tokens": 3116, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27568451000001915, "nonpadding_tokens": 14267, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27602714899990133, "nonpadding_tokens": 3784, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039594299994405446, "prefetch": {"data_wait_seconds": 0.018534135000962237, "consumed_batches": 1965, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2515, "tokens": 184999010, "loss_tokens": 57796, "loss": 2.6142328729496853, "lr": 0.0002992693399450792, "optimizer_lrs": [0.01995128932967195, 0.0002992693399450792], "gradient_norm": 0.4398541748523712, "seconds": 0.3557644039999559, "nonpadding_tokens_per_second": 162455.82568178227, "ranks": [{"compute_seconds": 0.352123784000014, "nonpadding_tokens": 16016, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34929224299992256, "nonpadding_tokens": 1991, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34995779299993046, "nonpadding_tokens": 2985, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35059172000001126, "nonpadding_tokens": 16005, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35321628500003044, "nonpadding_tokens": 9250, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35168289700004607, "nonpadding_tokens": 2756, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35342793399991024, "nonpadding_tokens": 4910, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3498765829999684, "nonpadding_tokens": 3883, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034314300000914955, "prefetch": {"data_wait_seconds": 0.018542259000994932, "consumed_batches": 1966, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2516, "tokens": 185043530, "loss_tokens": 44520, "loss": 2.5765937359613655, "lr": 0.0002992689459550819, "optimizer_lrs": [0.019951263063672127, 0.0002992689459550819], "gradient_norm": 0.7645701766014099, "seconds": 0.25618028399992454, "nonpadding_tokens_per_second": 173783.86542819632, "ranks": [{"compute_seconds": 0.25209758100004365, "nonpadding_tokens": 8515, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25390657299999475, "nonpadding_tokens": 3953, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2510217139999895, "nonpadding_tokens": 5583, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2519170780000195, "nonpadding_tokens": 4289, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25391514599994025, "nonpadding_tokens": 2831, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25255846300001394, "nonpadding_tokens": 8958, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24992313500001728, "nonpadding_tokens": 5243, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2507083320000447, "nonpadding_tokens": 5148, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036129099999016034, "prefetch": {"data_wait_seconds": 0.018550814001017102, "consumed_batches": 1967, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2517, "tokens": 185099837, "loss_tokens": 56307, "loss": 2.484380549931625, "lr": 0.00029926844750163335, "optimizer_lrs": [0.019951229833442226, 0.00029926844750163335], "gradient_norm": 0.9929409027099609, "seconds": 0.2707378970000036, "nonpadding_tokens_per_second": 207976.0558973362, "ranks": [{"compute_seconds": 0.26628965899999457, "nonpadding_tokens": 2469, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26845717100002275, "nonpadding_tokens": 4420, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2648312819999319, "nonpadding_tokens": 8356, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.265547978000086, "nonpadding_tokens": 2654, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.264144910999903, "nonpadding_tokens": 10223, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26689083200005825, "nonpadding_tokens": 10472, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26813775099992654, "nonpadding_tokens": 4277, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2649669630000062, "nonpadding_tokens": 13436, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003326859999788212, "prefetch": {"data_wait_seconds": 0.018558994001026008, "consumed_batches": 1968, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2518, "tokens": 185165589, "loss_tokens": 65752, "loss": 2.1122617943180435, "lr": 0.00029926786522261917, "optimizer_lrs": [0.01995119101484128, 0.00029926786522261917], "gradient_norm": 0.47962480783462524, "seconds": 0.2746087960000523, "nonpadding_tokens_per_second": 239438.79787444056, "ranks": [{"compute_seconds": 0.2721822659999589, "nonpadding_tokens": 14335, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2687000480000279, "nonpadding_tokens": 11846, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2692414220000501, "nonpadding_tokens": 5480, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2702702629999294, "nonpadding_tokens": 4808, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.267911832999971, "nonpadding_tokens": 7342, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2705579170000192, "nonpadding_tokens": 7332, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2718336200000522, "nonpadding_tokens": 2797, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27229788399995414, "nonpadding_tokens": 11812, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003871820000540538, "prefetch": {"data_wait_seconds": 0.018567885001019135, "consumed_batches": 1969, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2519, "tokens": 185242683, "loss_tokens": 77094, "loss": 1.9620041767193297, "lr": 0.0002992671822082036, "optimizer_lrs": [0.019951145480546908, 0.0002992671822082036], "gradient_norm": 0.4341820478439331, "seconds": 0.33162828700005775, "nonpadding_tokens_per_second": 232471.12210300256, "ranks": [{"compute_seconds": 0.3274558849999494, "nonpadding_tokens": 15286, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32542325999997956, "nonpadding_tokens": 9239, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3286840600000005, "nonpadding_tokens": 11992, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3266698230000884, "nonpadding_tokens": 4146, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32519936100004543, "nonpadding_tokens": 7709, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3281403229999569, "nonpadding_tokens": 9299, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32925144700004694, "nonpadding_tokens": 14995, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3260212339999953, "nonpadding_tokens": 4428, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003530799999680312, "prefetch": {"data_wait_seconds": 0.01857775100104675, "consumed_batches": 1970, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2520, "tokens": 185294656, "loss_tokens": 51973, "loss": 2.663686854232005, "lr": 0.00029926672157411254, "optimizer_lrs": [0.019951114771607506, 0.00029926672157411254], "gradient_norm": 0.4408062696456909, "seconds": 0.30610452299993085, "nonpadding_tokens_per_second": 169788.4091703269, "ranks": [{"compute_seconds": 0.3013872290000563, "nonpadding_tokens": 9614, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2995970889999171, "nonpadding_tokens": 1033, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3000344170000062, "nonpadding_tokens": 5800, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3004479740000079, "nonpadding_tokens": 2211, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29901947700000164, "nonpadding_tokens": 4678, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30008047600006194, "nonpadding_tokens": 4615, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2990005819999624, "nonpadding_tokens": 10653, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30346488100008173, "nonpadding_tokens": 13369, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0006813720000309331, "prefetch": {"data_wait_seconds": 0.018585148001037055, "consumed_batches": 1971, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2521, "tokens": 185374912, "loss_tokens": 80256, "loss": 2.6735845684434807, "lr": 0.00029926600998569187, "optimizer_lrs": [0.01995106733237946, 0.00029926600998569187], "gradient_norm": 0.42761874198913574, "seconds": 0.27186250500005826, "nonpadding_tokens_per_second": 295208.05011335714, "ranks": [{"compute_seconds": 0.2686263539999345, "nonpadding_tokens": 5539, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2693474390000574, "nonpadding_tokens": 11937, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2684141350000573, "nonpadding_tokens": 7406, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2672379450000335, "nonpadding_tokens": 5579, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26955994100001135, "nonpadding_tokens": 9086, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2667649210000036, "nonpadding_tokens": 11522, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2656644819999201, "nonpadding_tokens": 15017, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26629296499993416, "nonpadding_tokens": 14170, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033789600001910003, "prefetch": {"data_wait_seconds": 0.018593268000998364, "consumed_batches": 1972, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2522, "tokens": 185438958, "loss_tokens": 64046, "loss": 2.4915553859725823, "lr": 0.0002992654418760411, "optimizer_lrs": [0.01995102945840274, 0.0002992654418760411], "gradient_norm": 0.40095385909080505, "seconds": 0.34486239799991836, "nonpadding_tokens_per_second": 185714.6513259911, "ranks": [{"compute_seconds": 0.34143376200006514, "nonpadding_tokens": 13146, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33927008299997397, "nonpadding_tokens": 3356, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3418937360000882, "nonpadding_tokens": 9163, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33992090399999597, "nonpadding_tokens": 3045, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3425856669999803, "nonpadding_tokens": 1961, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3412998860000016, "nonpadding_tokens": 12721, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33885568599998805, "nonpadding_tokens": 16114, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3392183710000154, "nonpadding_tokens": 4540, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037856999995256047, "prefetch": {"data_wait_seconds": 0.018601481000999343, "consumed_batches": 1973, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2523, "tokens": 185497991, "loss_tokens": 59033, "loss": 2.3408729015974115, "lr": 0.0002992649180392806, "optimizer_lrs": [0.01995099453595204, 0.0002992649180392806], "gradient_norm": 0.5239827036857605, "seconds": 0.243291193999994, "nonpadding_tokens_per_second": 242643.38971513064, "ranks": [{"compute_seconds": 0.241004840999949, "nonpadding_tokens": 2649, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24059419900004286, "nonpadding_tokens": 2679, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23768342399989706, "nonpadding_tokens": 9821, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2387474120000661, "nonpadding_tokens": 6245, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23680507000005946, "nonpadding_tokens": 17462, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23942828500003088, "nonpadding_tokens": 5590, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24059833199999048, "nonpadding_tokens": 7346, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23741841399998975, "nonpadding_tokens": 7241, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003543029999946157, "prefetch": {"data_wait_seconds": 0.018610016001005647, "consumed_batches": 1974, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2524, "tokens": 185576085, "loss_tokens": 78094, "loss": 2.290610314172664, "lr": 0.0002992642247762384, "optimizer_lrs": [0.019950948318415894, 0.0002992642247762384], "gradient_norm": 4.028231143951416, "seconds": 0.2450713270000051, "nonpadding_tokens_per_second": 318658.2492369594, "ranks": [{"compute_seconds": 0.24244697200003884, "nonpadding_tokens": 1646, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24263930199992956, "nonpadding_tokens": 12245, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23980924200009213, "nonpadding_tokens": 12384, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24044332000005397, "nonpadding_tokens": 11890, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24275851499999135, "nonpadding_tokens": 11411, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24040210299995124, "nonpadding_tokens": 4380, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2388041049999856, "nonpadding_tokens": 10141, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23911541799998304, "nonpadding_tokens": 13997, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034814399998595036, "prefetch": {"data_wait_seconds": 0.018617906000898188, "consumed_batches": 1975, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2525, "tokens": 185670025, "loss_tokens": 93940, "loss": 2.0759537337662337, "lr": 0.0002992633904119139, "optimizer_lrs": [0.019950892694127597, 0.0002992633904119139], "gradient_norm": 1.9971847534179688, "seconds": 0.31476956900007735, "nonpadding_tokens_per_second": 298440.53952997254, "ranks": [{"compute_seconds": 0.3108304700000417, "nonpadding_tokens": 9446, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3087070330000188, "nonpadding_tokens": 11404, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30847628399999394, "nonpadding_tokens": 21112, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30891862600003606, "nonpadding_tokens": 12465, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3124454210000067, "nonpadding_tokens": 25142, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3103327019999824, "nonpadding_tokens": 8948, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3120406829999638, "nonpadding_tokens": 3173, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3077770730000111, "nonpadding_tokens": 2250, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004076239999903919, "prefetch": {"data_wait_seconds": 0.01862843300087036, "consumed_batches": 1976, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2526, "tokens": 185765102, "loss_tokens": 95077, "loss": 2.1202860576164584, "lr": 0.0002992625454687709, "optimizer_lrs": [0.01995083636458473, 0.0002992625454687709], "gradient_norm": 0.6329511404037476, "seconds": 0.3255171529999643, "nonpadding_tokens_per_second": 292079.84625010047, "ranks": [{"compute_seconds": 0.32170430299993313, "nonpadding_tokens": 14798, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3231392610000512, "nonpadding_tokens": 3033, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32001325500004896, "nonpadding_tokens": 4159, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3227871100000357, "nonpadding_tokens": 5884, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3195466339999484, "nonpadding_tokens": 3513, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32148215200004415, "nonpadding_tokens": 24531, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3229897890000757, "nonpadding_tokens": 17968, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31988724699999693, "nonpadding_tokens": 21191, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000398868999923252, "prefetch": {"data_wait_seconds": 0.018638754000861013, "consumed_batches": 1977, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2527, "tokens": 185822900, "loss_tokens": 57798, "loss": 2.381197500346033, "lr": 0.0002992620315856308, "optimizer_lrs": [0.019950802105708723, 0.0002992620315856308], "gradient_norm": 0.8031808733940125, "seconds": 0.27646784499995647, "nonpadding_tokens_per_second": 209058.67009600738, "ranks": [{"compute_seconds": 0.2737108479999506, "nonpadding_tokens": 9986, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2741519859999926, "nonpadding_tokens": 2712, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.271573568000008, "nonpadding_tokens": 3014, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2717110370000455, "nonpadding_tokens": 1227, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27023807800003397, "nonpadding_tokens": 19551, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2716552040000124, "nonpadding_tokens": 13403, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2702449240000533, "nonpadding_tokens": 2371, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27051927899992734, "nonpadding_tokens": 5534, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039523299994925765, "prefetch": {"data_wait_seconds": 0.01864776800091761, "consumed_batches": 1978, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2528, "tokens": 185878878, "loss_tokens": 55978, "loss": 2.2470849318035655, "lr": 0.000299261533713996, "optimizer_lrs": [0.019950768914266403, 0.000299261533713996], "gradient_norm": 0.7727083563804626, "seconds": 0.25562957100009953, "nonpadding_tokens_per_second": 218980.92533268855, "ranks": [{"compute_seconds": 0.25328332100002626, "nonpadding_tokens": 8438, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24997091599993837, "nonpadding_tokens": 2249, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2524139480000258, "nonpadding_tokens": 1919, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2527469219999148, "nonpadding_tokens": 16760, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24924532599993654, "nonpadding_tokens": 11046, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25206977199991343, "nonpadding_tokens": 5888, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25316038899995874, "nonpadding_tokens": 5497, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24968427499993595, "nonpadding_tokens": 4181, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041393200001493824, "prefetch": {"data_wait_seconds": 0.018657570000939927, "consumed_batches": 1979, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2529, "tokens": 185960465, "loss_tokens": 81587, "loss": 2.4051913141799552, "lr": 0.000299260807774563, "optimizer_lrs": [0.0199507205183042, 0.000299260807774563], "gradient_norm": 0.38956332206726074, "seconds": 0.3222474350000084, "nonpadding_tokens_per_second": 253181.2239250186, "ranks": [{"compute_seconds": 0.31942843499996343, "nonpadding_tokens": 20441, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3160141709998925, "nonpadding_tokens": 21164, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31920982599990566, "nonpadding_tokens": 4567, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31972578499994597, "nonpadding_tokens": 2733, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31570503199998257, "nonpadding_tokens": 5307, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3182838069999434, "nonpadding_tokens": 10213, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3157017750000932, "nonpadding_tokens": 4781, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31595125700005156, "nonpadding_tokens": 12381, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034739300008368446, "prefetch": {"data_wait_seconds": 0.018667056001007154, "consumed_batches": 1980, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2530, "tokens": 186021883, "loss_tokens": 61418, "loss": 2.333264220179752, "lr": 0.00029926026105888635, "optimizer_lrs": [0.019950684070592424, 0.00029926026105888635], "gradient_norm": 0.3886792063713074, "seconds": 0.2830198679999967, "nonpadding_tokens_per_second": 217009.49984190057, "ranks": [{"compute_seconds": 0.2801384550000421, "nonpadding_tokens": 4809, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2796031259999836, "nonpadding_tokens": 9029, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2769963880000432, "nonpadding_tokens": 10671, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2779169750000392, "nonpadding_tokens": 7212, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28028990099994644, "nonpadding_tokens": 3464, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2786965650000184, "nonpadding_tokens": 3824, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2756400160000112, "nonpadding_tokens": 20670, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2806705419999389, "nonpadding_tokens": 1739, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039996500004235713, "prefetch": {"data_wait_seconds": 0.018674986001045113, "consumed_batches": 1981, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2531, "tokens": 186076016, "loss_tokens": 54133, "loss": 2.406666653196756, "lr": 0.00029925977902392763, "optimizer_lrs": [0.01995065193492851, 0.00029925977902392763], "gradient_norm": 0.4357716739177704, "seconds": 0.262668690000055, "nonpadding_tokens_per_second": 206088.51401356084, "ranks": [{"compute_seconds": 0.2600166540000828, "nonpadding_tokens": 3734, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2567155819999698, "nonpadding_tokens": 7851, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25673969899992244, "nonpadding_tokens": 7949, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25747727999998915, "nonpadding_tokens": 11185, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2602775100000372, "nonpadding_tokens": 12664, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2572136799999498, "nonpadding_tokens": 3374, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25582862600003864, "nonpadding_tokens": 4720, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2567938500000082, "nonpadding_tokens": 2656, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003967229999943811, "prefetch": {"data_wait_seconds": 0.018684031001001244, "consumed_batches": 1982, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2532, "tokens": 186125456, "loss_tokens": 49440, "loss": 2.584087245904126, "lr": 0.00029925933864165277, "optimizer_lrs": [0.019950622576110185, 0.00029925933864165277], "gradient_norm": 0.436294823884964, "seconds": 0.2626587439999639, "nonpadding_tokens_per_second": 188229.0276999375, "ranks": [{"compute_seconds": 0.2601994830000649, "nonpadding_tokens": 4913, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2572164039999052, "nonpadding_tokens": 6563, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26012437899998986, "nonpadding_tokens": 6993, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25792182800000774, "nonpadding_tokens": 18009, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2567211010000392, "nonpadding_tokens": 3604, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2591406189999361, "nonpadding_tokens": 4323, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2563081340000508, "nonpadding_tokens": 2904, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25699279099990235, "nonpadding_tokens": 2131, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040374799993969646, "prefetch": {"data_wait_seconds": 0.018694215001076486, "consumed_batches": 1983, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2533, "tokens": 186175610, "loss_tokens": 50154, "loss": 2.574944171950393, "lr": 0.00029925889176604566, "optimizer_lrs": [0.019950592784403046, 0.00029925889176604566], "gradient_norm": 0.5628550052642822, "seconds": 0.27959968400000434, "nonpadding_tokens_per_second": 179377.88513380158, "ranks": [{"compute_seconds": 0.2771332010000833, "nonpadding_tokens": 8175, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2769980770000302, "nonpadding_tokens": 2566, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.274471923999954, "nonpadding_tokens": 3482, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27664277100006984, "nonpadding_tokens": 9049, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27721726099991884, "nonpadding_tokens": 6412, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27420359099994585, "nonpadding_tokens": 2523, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27689394200001516, "nonpadding_tokens": 4660, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27353038799992646, "nonpadding_tokens": 13287, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003823650000640555, "prefetch": {"data_wait_seconds": 0.018703050001093402, "consumed_batches": 1984, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2534, "tokens": 186246589, "loss_tokens": 70979, "loss": 2.291183836064188, "lr": 0.0002992582591085569, "optimizer_lrs": [0.019950550607237126, 0.0002992582591085569], "gradient_norm": 0.410158634185791, "seconds": 0.2913771050000378, "nonpadding_tokens_per_second": 243598.41175575822, "ranks": [{"compute_seconds": 0.2889903279999544, "nonpadding_tokens": 2801, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28882567599998765, "nonpadding_tokens": 3617, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2861102150000079, "nonpadding_tokens": 4542, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28867365100006737, "nonpadding_tokens": 15289, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2854022610000584, "nonpadding_tokens": 12937, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2867491700000073, "nonpadding_tokens": 9909, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2890904870000668, "nonpadding_tokens": 10341, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28546356400011064, "nonpadding_tokens": 11543, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003589170000850572, "prefetch": {"data_wait_seconds": 0.01871087100114721, "consumed_batches": 1985, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2535, "tokens": 186290938, "loss_tokens": 44349, "loss": 2.2573590653115065, "lr": 0.0002992578636757473, "optimizer_lrs": [0.01995052424504982, 0.0002992578636757473], "gradient_norm": 0.4499264657497406, "seconds": 0.2805240519999188, "nonpadding_tokens_per_second": 158093.39585617007, "ranks": [{"compute_seconds": 0.27828685600002245, "nonpadding_tokens": 1572, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27800284999989344, "nonpadding_tokens": 7048, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27696030799995697, "nonpadding_tokens": 3815, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.275358730999983, "nonpadding_tokens": 8970, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2738860399999794, "nonpadding_tokens": 6884, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27690013599999475, "nonpadding_tokens": 4079, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2777338350000491, "nonpadding_tokens": 3082, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27477741999996397, "nonpadding_tokens": 8899, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004093549999879542, "prefetch": {"data_wait_seconds": 0.018721290001238344, "consumed_batches": 1986, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2536, "tokens": 186343753, "loss_tokens": 52815, "loss": 2.453406051784531, "lr": 0.0002992573926197202, "optimizer_lrs": [0.01995049284131468, 0.0002992573926197202], "gradient_norm": 0.44922274351119995, "seconds": 0.2758516009999994, "nonpadding_tokens_per_second": 191461.64027520042, "ranks": [{"compute_seconds": 0.27295720100005383, "nonpadding_tokens": 11142, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.269451097000001, "nonpadding_tokens": 3347, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2720547970000098, "nonpadding_tokens": 8722, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2701031139999941, "nonpadding_tokens": 9459, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2680499159999954, "nonpadding_tokens": 1880, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2709123350000482, "nonpadding_tokens": 6658, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2678270889999794, "nonpadding_tokens": 2735, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2734558779999361, "nonpadding_tokens": 8872, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003967239999838057, "prefetch": {"data_wait_seconds": 0.018730869001231554, "consumed_batches": 1987, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2537, "tokens": 186402274, "loss_tokens": 58521, "loss": 2.2437535777754993, "lr": 0.0002992568704979235, "optimizer_lrs": [0.019950458033194902, 0.0002992568704979235], "gradient_norm": 0.4016789495944977, "seconds": 0.3274900430000116, "nonpadding_tokens_per_second": 178695.50922498712, "ranks": [{"compute_seconds": 0.3228001930000346, "nonpadding_tokens": 2050, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32468393800002104, "nonpadding_tokens": 14045, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3213455200000226, "nonpadding_tokens": 6741, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3249150249999957, "nonpadding_tokens": 16375, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3205453530000568, "nonpadding_tokens": 6438, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3227166930000749, "nonpadding_tokens": 2135, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32517720999999256, "nonpadding_tokens": 3098, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3208848870000338, "nonpadding_tokens": 7639, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036133200001131627, "prefetch": {"data_wait_seconds": 0.018739331001214055, "consumed_batches": 1988, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2538, "tokens": 186458998, "loss_tokens": 56724, "loss": 2.323576385216134, "lr": 0.0002992563642342449, "optimizer_lrs": [0.019950424282282993, 0.0002992563642342449], "gradient_norm": 0.42055657505989075, "seconds": 0.24501248399997166, "nonpadding_tokens_per_second": 231514.73375538923, "ranks": [{"compute_seconds": 0.24272823399996923, "nonpadding_tokens": 4698, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24106944300001487, "nonpadding_tokens": 6438, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24176796300002934, "nonpadding_tokens": 10691, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23859940499994536, "nonpadding_tokens": 2753, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23722413699999834, "nonpadding_tokens": 7940, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24034607900000537, "nonpadding_tokens": 6742, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2424601679999796, "nonpadding_tokens": 3619, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2380672819999745, "nonpadding_tokens": 13843, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003498410000020158, "prefetch": {"data_wait_seconds": 0.018747886001122538, "consumed_batches": 1989, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2539, "tokens": 186524971, "loss_tokens": 65973, "loss": 2.3504883152956513, "lr": 0.00029925577520667376, "optimizer_lrs": [0.019950385013778254, 0.00029925577520667376], "gradient_norm": 0.8423706293106079, "seconds": 0.31673005599998305, "nonpadding_tokens_per_second": 208294.09381976534, "ranks": [{"compute_seconds": 0.31133720100001483, "nonpadding_tokens": 11141, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3135769130000199, "nonpadding_tokens": 4439, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31345137599998907, "nonpadding_tokens": 5640, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31140417999995407, "nonpadding_tokens": 1781, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3136149999999134, "nonpadding_tokens": 2894, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3127829089999068, "nonpadding_tokens": 6939, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31346021399997426, "nonpadding_tokens": 20490, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3144511520000606, "nonpadding_tokens": 12649, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036096300004828663, "prefetch": {"data_wait_seconds": 0.018756881001039005, "consumed_batches": 1990, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2540, "tokens": 186613802, "loss_tokens": 88831, "loss": 2.3558280611498237, "lr": 0.0002992549817283729, "optimizer_lrs": [0.019950332115224862, 0.0002992549817283729], "gradient_norm": 0.4604237675666809, "seconds": 0.33088200000008783, "nonpadding_tokens_per_second": 268467.3085872801, "ranks": [{"compute_seconds": 0.32528055499994935, "nonpadding_tokens": 17958, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32766349499991065, "nonpadding_tokens": 15665, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32454924299997856, "nonpadding_tokens": 8747, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3271691639999972, "nonpadding_tokens": 11314, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32325054799991904, "nonpadding_tokens": 6837, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3260130979999758, "nonpadding_tokens": 13210, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3237044779999678, "nonpadding_tokens": 9565, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3285072690000561, "nonpadding_tokens": 5535, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039290999995955644, "prefetch": {"data_wait_seconds": 0.018765792001048, "consumed_batches": 1991, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2541, "tokens": 186716766, "loss_tokens": 102964, "loss": 2.769917155510664, "lr": 0.000299254061480232, "optimizer_lrs": [0.0199502707653488, 0.000299254061480232], "gradient_norm": 0.5755529403686523, "seconds": 0.25384501199994247, "nonpadding_tokens_per_second": 405617.5821174825, "ranks": [{"compute_seconds": 0.2516149929999756, "nonpadding_tokens": 2309, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.250910971000053, "nonpadding_tokens": 2176, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24763732899998558, "nonpadding_tokens": 12058, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2485642449998977, "nonpadding_tokens": 27225, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2513487679999571, "nonpadding_tokens": 19434, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24849272699998437, "nonpadding_tokens": 4161, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.250986836999914, "nonpadding_tokens": 19658, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24771122699996795, "nonpadding_tokens": 15943, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037103099998603284, "prefetch": {"data_wait_seconds": 0.018775587001144345, "consumed_batches": 1992, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2542, "tokens": 186757480, "loss_tokens": 40714, "loss": 2.8402599535786215, "lr": 0.00029925369743966535, "optimizer_lrs": [0.01995024649597769, 0.00029925369743966535], "gradient_norm": 0.529099702835083, "seconds": 0.2379351450000513, "nonpadding_tokens_per_second": 171113.85541632035, "ranks": [{"compute_seconds": 0.23558331799995358, "nonpadding_tokens": 2574, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2352660899999819, "nonpadding_tokens": 6825, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23414822899997034, "nonpadding_tokens": 3225, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23273633799999516, "nonpadding_tokens": 6207, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23544272200001615, "nonpadding_tokens": 1941, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23230295000007573, "nonpadding_tokens": 2043, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23565171899997495, "nonpadding_tokens": 4042, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23546196899997085, "nonpadding_tokens": 13857, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003662980000171956, "prefetch": {"data_wait_seconds": 0.01878429600112952, "consumed_batches": 1993, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2543, "tokens": 186808745, "loss_tokens": 51265, "loss": 2.385417225446211, "lr": 0.00029925323893231775, "optimizer_lrs": [0.019950215928821183, 0.00029925323893231775], "gradient_norm": 0.4375043511390686, "seconds": 0.2779525079999985, "nonpadding_tokens_per_second": 184437.98319675634, "ranks": [{"compute_seconds": 0.274379993000025, "nonpadding_tokens": 10858, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2717567600000166, "nonpadding_tokens": 10166, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2727506710000398, "nonpadding_tokens": 1882, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2734522640000705, "nonpadding_tokens": 7019, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27153812700009894, "nonpadding_tokens": 9404, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2747347249999166, "nonpadding_tokens": 2181, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.275502715000016, "nonpadding_tokens": 3597, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2756253170000491, "nonpadding_tokens": 6158, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003580250000823071, "prefetch": {"data_wait_seconds": 0.018793923001069857, "consumed_batches": 1994, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2544, "tokens": 186860606, "loss_tokens": 51861, "loss": 2.1239778868031856, "lr": 0.000299252774951554, "optimizer_lrs": [0.01995018499677027, 0.000299252774951554], "gradient_norm": 0.4414632022380829, "seconds": 0.322988275000057, "nonpadding_tokens_per_second": 160566.20011977476, "ranks": [{"compute_seconds": 0.31806123800004116, "nonpadding_tokens": 2171, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3206510549999848, "nonpadding_tokens": 11358, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31923219999998764, "nonpadding_tokens": 14553, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32025833400007286, "nonpadding_tokens": 3888, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32044268899994677, "nonpadding_tokens": 2238, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3192691750000449, "nonpadding_tokens": 6704, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3163873449999528, "nonpadding_tokens": 2631, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.316633754999998, "nonpadding_tokens": 8318, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039139299997259513, "prefetch": {"data_wait_seconds": 0.01880209300107083, "consumed_batches": 1995, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2545, "tokens": 186907795, "loss_tokens": 47189, "loss": 2.25272243001547, "lr": 0.00029925235264454655, "optimizer_lrs": [0.01995015684296977, 0.00029925235264454655], "gradient_norm": 0.4461252987384796, "seconds": 0.2499503440000126, "nonpadding_tokens_per_second": 188793.49891991995, "ranks": [{"compute_seconds": 0.24698522100004539, "nonpadding_tokens": 3448, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24342946000001575, "nonpadding_tokens": 2472, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24409364199993888, "nonpadding_tokens": 4813, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24417959499999142, "nonpadding_tokens": 24512, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24293291399999362, "nonpadding_tokens": 5071, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24426502400001482, "nonpadding_tokens": 1568, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24289451200002077, "nonpadding_tokens": 2632, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24758710399999018, "nonpadding_tokens": 2673, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004086829999323527, "prefetch": {"data_wait_seconds": 0.018811093001048107, "consumed_batches": 1996, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2546, "tokens": 186965968, "loss_tokens": 58173, "loss": 2.1609391384319188, "lr": 0.0002992518318750455, "optimizer_lrs": [0.019950122125003036, 0.0002992518318750455], "gradient_norm": 0.3994443118572235, "seconds": 0.19400927399999546, "nonpadding_tokens_per_second": 299846.4908435324, "ranks": [{"compute_seconds": 0.19175372999995943, "nonpadding_tokens": 5043, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.1912389520000488, "nonpadding_tokens": 5743, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.18839837099994838, "nonpadding_tokens": 4192, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.1910717890000342, "nonpadding_tokens": 5082, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1915326539999569, "nonpadding_tokens": 11921, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1885970719999932, "nonpadding_tokens": 7683, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.18707389299993338, "nonpadding_tokens": 8817, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.18827562199999193, "nonpadding_tokens": 9692, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003517079999255657, "prefetch": {"data_wait_seconds": 0.018819709000993612, "consumed_batches": 1997, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2547, "tokens": 187024846, "loss_tokens": 58878, "loss": 2.603311349315534, "lr": 0.0002992513046102239, "optimizer_lrs": [0.01995008697401493, 0.0002992513046102239], "gradient_norm": 0.444511353969574, "seconds": 0.27733253099995636, "nonpadding_tokens_per_second": 212301.09496245597, "ranks": [{"compute_seconds": 0.23464823800009071, "nonpadding_tokens": 4250, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24165571800006092, "nonpadding_tokens": 2809, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24133393800002523, "nonpadding_tokens": 7225, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2449912060000088, "nonpadding_tokens": 17794, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24087964099999226, "nonpadding_tokens": 11971, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24313507199997275, "nonpadding_tokens": 7492, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24552861999995912, "nonpadding_tokens": 3217, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24120273799996994, "nonpadding_tokens": 4120, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.026344461999997293, "prefetch": {"data_wait_seconds": 0.018827610000997197, "consumed_batches": 1998, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2548, "tokens": 187115338, "loss_tokens": 90492, "loss": 2.22926888564735, "lr": 0.00029925049387433144, "optimizer_lrs": [0.01995003292495543, 0.00029925049387433144], "gradient_norm": 0.4134824275970459, "seconds": 0.3440263779999668, "nonpadding_tokens_per_second": 263037.96972221916, "ranks": [{"compute_seconds": 0.3406711819999373, "nonpadding_tokens": 13409, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3415629699999272, "nonpadding_tokens": 5269, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33831852899993464, "nonpadding_tokens": 9352, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3387160550000772, "nonpadding_tokens": 16994, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3413560610000559, "nonpadding_tokens": 5837, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33867719099998794, "nonpadding_tokens": 18215, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33708614700003636, "nonpadding_tokens": 14629, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34165324399998553, "nonpadding_tokens": 6787, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004449780000186365, "prefetch": {"data_wait_seconds": 0.01883729200103801, "consumed_batches": 1999, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2549, "tokens": 187165309, "loss_tokens": 49971, "loss": 1.973084232604911, "lr": 0.0002992500459866644, "optimizer_lrs": [0.019950003065777627, 0.0002992500459866644], "gradient_norm": 0.9901286363601685, "seconds": 0.26398820599990813, "nonpadding_tokens_per_second": 189292.54741030888, "ranks": [{"compute_seconds": 0.25987682699997094, "nonpadding_tokens": 2670, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2579341230000409, "nonpadding_tokens": 7630, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.258739478999928, "nonpadding_tokens": 3086, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25898877500003437, "nonpadding_tokens": 4858, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2617163759999812, "nonpadding_tokens": 3173, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25894476999997096, "nonpadding_tokens": 2721, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26129044399999657, "nonpadding_tokens": 19564, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25814680399992085, "nonpadding_tokens": 6269, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003681600001073093, "prefetch": {"data_wait_seconds": 0.01884593300106019, "consumed_batches": 2000, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2550, "tokens": 187198868, "loss_tokens": 33559, "loss": 2.2426752138025567, "lr": 0.00029924974512408116, "optimizer_lrs": [0.01994998300827208, 0.00029924974512408116], "gradient_norm": 0.4385640621185303, "seconds": 0.2061323519999405, "nonpadding_tokens_per_second": 162803.168325609, "ranks": [{"compute_seconds": 0.2034204870000167, "nonpadding_tokens": 3714, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2028090630000179, "nonpadding_tokens": 6002, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20228147099999205, "nonpadding_tokens": 4517, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20084380900004817, "nonpadding_tokens": 2735, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2030044989999169, "nonpadding_tokens": 7383, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2020358889999443, "nonpadding_tokens": 4367, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19920840800000406, "nonpadding_tokens": 2017, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2038611039999978, "nonpadding_tokens": 2824, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036522999994303973, "prefetch": {"data_wait_seconds": 0.018853509001090174, "consumed_batches": 2001, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2551, "tokens": 187275282, "loss_tokens": 76414, "loss": 2.2163559688015284, "lr": 0.0002992490598340889, "optimizer_lrs": [0.01994993732227259, 0.0002992490598340889], "gradient_norm": 0.5318334698677063, "seconds": 0.4102455129999498, "nonpadding_tokens_per_second": 186264.07255795956, "ranks": [{"compute_seconds": 0.4057645480000929, "nonpadding_tokens": 9173, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.40414917999999034, "nonpadding_tokens": 4315, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.40665798599991376, "nonpadding_tokens": 13989, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.40747947300008036, "nonpadding_tokens": 3559, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4079128959999707, "nonpadding_tokens": 11122, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4062818029999562, "nonpadding_tokens": 24528, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.40737269900000683, "nonpadding_tokens": 5576, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.40417597800001204, "nonpadding_tokens": 4152, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039266299995688314, "prefetch": {"data_wait_seconds": 0.018863025001067513, "consumed_batches": 2002, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2552, "tokens": 187353817, "loss_tokens": 78535, "loss": 2.1581561962819125, "lr": 0.000299248355197644, "optimizer_lrs": [0.0199498903465096, 0.000299248355197644], "gradient_norm": 0.7792814373970032, "seconds": 0.26049983300003987, "nonpadding_tokens_per_second": 301478.1203333362, "ranks": [{"compute_seconds": 0.2576707810000016, "nonpadding_tokens": 4813, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25456220600005963, "nonpadding_tokens": 1480, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25547984599995743, "nonpadding_tokens": 3426, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2564008059999878, "nonpadding_tokens": 6661, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2542172700000265, "nonpadding_tokens": 17646, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25673354299999573, "nonpadding_tokens": 10824, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25821261500004766, "nonpadding_tokens": 19636, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2549435270000231, "nonpadding_tokens": 14049, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033985199991093396, "prefetch": {"data_wait_seconds": 0.018871286001129306, "consumed_batches": 2003, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2553, "tokens": 187430813, "loss_tokens": 76996, "loss": 2.552146986531768, "lr": 0.000299247664049614, "optimizer_lrs": [0.01994984426997427, 0.000299247664049614], "gradient_norm": 1.1936001777648926, "seconds": 0.2635642910000797, "nonpadding_tokens_per_second": 292133.656300112, "ranks": [{"compute_seconds": 0.2595143089999965, "nonpadding_tokens": 8131, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.257133060000001, "nonpadding_tokens": 2133, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2603349880000678, "nonpadding_tokens": 15253, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.258390421999934, "nonpadding_tokens": 16857, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26123898599996664, "nonpadding_tokens": 19520, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2580775830000448, "nonpadding_tokens": 8291, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25702714900000956, "nonpadding_tokens": 2605, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25729191600009926, "nonpadding_tokens": 4206, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033683800006656384, "prefetch": {"data_wait_seconds": 0.01887885700109848, "consumed_batches": 2004, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2554, "tokens": 187474174, "loss_tokens": 43361, "loss": 2.409316551740043, "lr": 0.00029924727468388576, "optimizer_lrs": [0.019949818312259053, 0.00029924727468388576], "gradient_norm": 0.4480527937412262, "seconds": 0.30841386700001294, "nonpadding_tokens_per_second": 140593.5486033064, "ranks": [{"compute_seconds": 0.30596031400000356, "nonpadding_tokens": 6736, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30388210899991464, "nonpadding_tokens": 5165, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30063077099998736, "nonpadding_tokens": 2848, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30080619800003205, "nonpadding_tokens": 6497, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.305419962999963, "nonpadding_tokens": 9955, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30242344600003435, "nonpadding_tokens": 4131, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30538544700004877, "nonpadding_tokens": 1632, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3060348190000468, "nonpadding_tokens": 6397, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033531099995798286, "prefetch": {"data_wait_seconds": 0.01888678200107563, "consumed_batches": 2005, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2555, "tokens": 187584225, "loss_tokens": 110051, "loss": 2.2121811705482, "lr": 0.000299246286015653, "optimizer_lrs": [0.019949752401043533, 0.000299246286015653], "gradient_norm": 0.5533168911933899, "seconds": 0.2895458120000285, "nonpadding_tokens_per_second": 380081.4774001607, "ranks": [{"compute_seconds": 0.28635373999998137, "nonpadding_tokens": 5866, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2872051019999162, "nonpadding_tokens": 17054, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28437874400003693, "nonpadding_tokens": 13565, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2868878140000106, "nonpadding_tokens": 26467, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28362545900006353, "nonpadding_tokens": 17022, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28468702700001813, "nonpadding_tokens": 18651, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28694715000006, "nonpadding_tokens": 5059, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28402669599995534, "nonpadding_tokens": 6367, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043796399995699176, "prefetch": {"data_wait_seconds": 0.018896049001071447, "consumed_batches": 2006, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2556, "tokens": 187646207, "loss_tokens": 61982, "loss": 1.8587679689264625, "lr": 0.00029924572890140035, "optimizer_lrs": [0.019949715260093356, 0.00029924572890140035], "gradient_norm": 0.6630295515060425, "seconds": 0.3147474099999954, "nonpadding_tokens_per_second": 196926.1637450834, "ranks": [{"compute_seconds": 0.31095337199997175, "nonpadding_tokens": 4951, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3082971369999541, "nonpadding_tokens": 16281, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3087095479999107, "nonpadding_tokens": 14286, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3092368800000713, "nonpadding_tokens": 2651, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3119348100000252, "nonpadding_tokens": 11053, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3088353260000076, "nonpadding_tokens": 2778, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30792172399992523, "nonpadding_tokens": 4230, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.312421234999988, "nonpadding_tokens": 5752, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003657370000382798, "prefetch": {"data_wait_seconds": 0.018906281001093816, "consumed_batches": 2007, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2557, "tokens": 187693220, "loss_tokens": 47013, "loss": 2.4487912585348734, "lr": 0.0002992453061964798, "optimizer_lrs": [0.01994968707976532, 0.0002992453061964798], "gradient_norm": 0.4084227979183197, "seconds": 0.23356909199992515, "nonpadding_tokens_per_second": 201280.91263040516, "ranks": [{"compute_seconds": 0.2306953089999979, "nonpadding_tokens": 3401, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23017226399997526, "nonpadding_tokens": 2526, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22650569300003554, "nonpadding_tokens": 3438, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22717816500005483, "nonpadding_tokens": 10966, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23109576899992135, "nonpadding_tokens": 6265, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22797673399998075, "nonpadding_tokens": 10776, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2310355569999274, "nonpadding_tokens": 4685, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2261690960000351, "nonpadding_tokens": 4956, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036160000001927983, "prefetch": {"data_wait_seconds": 0.01891618900113201, "consumed_batches": 2008, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2558, "tokens": 187761114, "loss_tokens": 67894, "loss": 2.3833109792470615, "lr": 0.0002992446955371907, "optimizer_lrs": [0.01994964636914605, 0.0002992446955371907], "gradient_norm": 0.4274848997592926, "seconds": 0.25796765200004756, "nonpadding_tokens_per_second": 263188.0372349456, "ranks": [{"compute_seconds": 0.25553991200001747, "nonpadding_tokens": 7477, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2549570339999718, "nonpadding_tokens": 4583, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25194704299997284, "nonpadding_tokens": 5375, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2520459859999846, "nonpadding_tokens": 2017, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25073545500004, "nonpadding_tokens": 6765, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2533999040000481, "nonpadding_tokens": 12389, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.254405492999922, "nonpadding_tokens": 9498, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2513105790000054, "nonpadding_tokens": 19790, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043387599998823134, "prefetch": {"data_wait_seconds": 0.01892646700112266, "consumed_batches": 2009, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2559, "tokens": 187843547, "loss_tokens": 82433, "loss": 2.4152140981160457, "lr": 0.000299243953778699, "optimizer_lrs": [0.019949596918579934, 0.000299243953778699], "gradient_norm": 0.4001298248767853, "seconds": 0.3227023870000494, "nonpadding_tokens_per_second": 255445.8947958987, "ranks": [{"compute_seconds": 0.3195317360000445, "nonpadding_tokens": 13484, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32000126699995235, "nonpadding_tokens": 6173, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3168324460000349, "nonpadding_tokens": 4066, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3174158440000383, "nonpadding_tokens": 15547, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3157426540000188, "nonpadding_tokens": 19751, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3173325840000416, "nonpadding_tokens": 12268, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31971629899999243, "nonpadding_tokens": 8030, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32037838899998405, "nonpadding_tokens": 3114, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037493900003937597, "prefetch": {"data_wait_seconds": 0.01893649900110006, "consumed_batches": 2010, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2560, "tokens": 187915775, "loss_tokens": 72228, "loss": 2.341774268635432, "lr": 0.00029924330354966477, "optimizer_lrs": [0.019949553569977654, 0.00029924330354966477], "gradient_norm": 0.4090577960014343, "seconds": 0.28679121600009694, "nonpadding_tokens_per_second": 251848.71770959537, "ranks": [{"compute_seconds": 0.28423162600006435, "nonpadding_tokens": 2082, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2805275519999668, "nonpadding_tokens": 9938, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2808857369999487, "nonpadding_tokens": 9282, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2817810929999496, "nonpadding_tokens": 6107, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28025739300005625, "nonpadding_tokens": 17583, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28317198800004917, "nonpadding_tokens": 3924, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28365676699991127, "nonpadding_tokens": 10855, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2844886809999707, "nonpadding_tokens": 12457, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003647500000170112, "prefetch": {"data_wait_seconds": 0.018945378001106405, "consumed_batches": 2011, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2561, "tokens": 187979415, "loss_tokens": 63640, "loss": 2.497494451209931, "lr": 0.00029924273040271256, "optimizer_lrs": [0.019949515360180838, 0.00029924273040271256], "gradient_norm": 0.43417102098464966, "seconds": 0.3046610920000603, "nonpadding_tokens_per_second": 208887.8484029966, "ranks": [{"compute_seconds": 0.299289908999981, "nonpadding_tokens": 4240, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2976157000000512, "nonpadding_tokens": 19384, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2998197240000309, "nonpadding_tokens": 2385, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29833467200000996, "nonpadding_tokens": 2575, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30171760399991854, "nonpadding_tokens": 5409, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2997506419999354, "nonpadding_tokens": 17009, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2964140750000297, "nonpadding_tokens": 10407, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29764137400002255, "nonpadding_tokens": 2231, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0006335089999538468, "prefetch": {"data_wait_seconds": 0.01895156000109921, "consumed_batches": 2012, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2562, "tokens": 188035475, "loss_tokens": 56060, "loss": 2.342179974580806, "lr": 0.0002992422253426034, "optimizer_lrs": [0.019949481689506894, 0.0002992422253426034], "gradient_norm": 0.4852793216705322, "seconds": 0.24545756999998503, "nonpadding_tokens_per_second": 228389.77832300475, "ranks": [{"compute_seconds": 0.240329738000014, "nonpadding_tokens": 2443, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24314131799997085, "nonpadding_tokens": 1519, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2400140620000002, "nonpadding_tokens": 14059, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24222198500001468, "nonpadding_tokens": 2896, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23896407699999145, "nonpadding_tokens": 16701, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.241122732000008, "nonpadding_tokens": 2884, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23872062300006291, "nonpadding_tokens": 9716, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24303314799999498, "nonpadding_tokens": 5842, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003600039999582805, "prefetch": {"data_wait_seconds": 0.018959210001028737, "consumed_batches": 2013, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2563, "tokens": 188107120, "loss_tokens": 71645, "loss": 2.1356545292762927, "lr": 0.0002992415796285049, "optimizer_lrs": [0.019949438641900328, 0.0002992415796285049], "gradient_norm": 0.45172742009162903, "seconds": 0.30573045300002377, "nonpadding_tokens_per_second": 234340.41096322986, "ranks": [{"compute_seconds": 0.30081890200006, "nonpadding_tokens": 21201, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29925196600004256, "nonpadding_tokens": 6691, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29997306300003856, "nonpadding_tokens": 8762, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3027463839999882, "nonpadding_tokens": 5981, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30283243600001697, "nonpadding_tokens": 10260, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30158198499998434, "nonpadding_tokens": 4276, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3025304550000101, "nonpadding_tokens": 10132, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3034222280000449, "nonpadding_tokens": 4342, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003501969999888388, "prefetch": {"data_wait_seconds": 0.018967585001021803, "consumed_batches": 2014, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2564, "tokens": 188168921, "loss_tokens": 61801, "loss": 2.1396632437177394, "lr": 0.00029924102241502405, "optimizer_lrs": [0.01994940149433494, 0.00029924102241502405], "gradient_norm": 0.5392410755157471, "seconds": 0.2903211889999966, "nonpadding_tokens_per_second": 212871.13149705625, "ranks": [{"compute_seconds": 0.2863664809999591, "nonpadding_tokens": 11334, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28797476900001584, "nonpadding_tokens": 5823, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28477951100001064, "nonpadding_tokens": 12742, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2864963560000433, "nonpadding_tokens": 11534, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2842096679999031, "nonpadding_tokens": 8319, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28447251199997936, "nonpadding_tokens": 6411, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2866443419999314, "nonpadding_tokens": 3544, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28774648700004946, "nonpadding_tokens": 2094, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004007240000873935, "prefetch": {"data_wait_seconds": 0.018974591001054364, "consumed_batches": 2015, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2565, "tokens": 188257316, "loss_tokens": 88395, "loss": 2.6471371401097348, "lr": 0.00029924022506866297, "optimizer_lrs": [0.019949348337910865, 0.00029924022506866297], "gradient_norm": 0.390964150428772, "seconds": 0.4460229219999974, "nonpadding_tokens_per_second": 198184.8816281252, "ranks": [{"compute_seconds": 0.441663702000028, "nonpadding_tokens": 25356, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.43932544799997686, "nonpadding_tokens": 12575, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.43983248999995794, "nonpadding_tokens": 3819, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.44236692099991615, "nonpadding_tokens": 13804, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4432610429998931, "nonpadding_tokens": 6519, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4401555870000493, "nonpadding_tokens": 3609, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4431294190000017, "nonpadding_tokens": 15159, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4434315199999901, "nonpadding_tokens": 7554, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003437919999669248, "prefetch": {"data_wait_seconds": 0.01898257600107911, "consumed_batches": 2016, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2566, "tokens": 188302423, "loss_tokens": 45107, "loss": 2.6075739006695193, "lr": 0.0002992398180306992, "optimizer_lrs": [0.019949321202046615, 0.0002992398180306992], "gradient_norm": 0.46987324953079224, "seconds": 0.23567629400008627, "nonpadding_tokens_per_second": 191393.87858832968, "ranks": [{"compute_seconds": 0.2328771819999247, "nonpadding_tokens": 4483, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22899025600008827, "nonpadding_tokens": 6359, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22909434400003192, "nonpadding_tokens": 3749, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2290220350000709, "nonpadding_tokens": 9212, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23294733299997006, "nonpadding_tokens": 1609, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2306923450000795, "nonpadding_tokens": 2163, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22743634899995868, "nonpadding_tokens": 12046, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22836466100000052, "nonpadding_tokens": 5486, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036597899998014327, "prefetch": {"data_wait_seconds": 0.018991908001069646, "consumed_batches": 2017, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2567, "tokens": 188348712, "loss_tokens": 46289, "loss": 2.73844622912571, "lr": 0.0002992394002135618, "optimizer_lrs": [0.019949293347570788, 0.0002992394002135618], "gradient_norm": 0.553181529045105, "seconds": 0.27646642800004884, "nonpadding_tokens_per_second": 167430.81731425208, "ranks": [{"compute_seconds": 0.2117029820000198, "nonpadding_tokens": 4454, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21267333399998734, "nonpadding_tokens": 2121, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2155312900000581, "nonpadding_tokens": 6569, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21586838700000044, "nonpadding_tokens": 9680, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21257752200006053, "nonpadding_tokens": 1729, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2152681740000162, "nonpadding_tokens": 4486, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21230617699995946, "nonpadding_tokens": 5402, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21279366900000696, "nonpadding_tokens": 11848, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.03507005400001617, "prefetch": {"data_wait_seconds": 0.019001411001070778, "consumed_batches": 2018, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2568, "tokens": 188382156, "loss_tokens": 33444, "loss": 2.0847157748774072, "lr": 0.0002992390982676699, "optimizer_lrs": [0.019949273217844662, 0.0002992390982676699], "gradient_norm": 0.47268053889274597, "seconds": 0.21501010500003304, "nonpadding_tokens_per_second": 155546.17770171716, "ranks": [{"compute_seconds": 0.21183178800004043, "nonpadding_tokens": 3107, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2081986059999963, "nonpadding_tokens": 5297, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20911677000003692, "nonpadding_tokens": 2242, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20996639399993455, "nonpadding_tokens": 3503, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2085665900000322, "nonpadding_tokens": 2465, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2107116050000286, "nonpadding_tokens": 3080, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2122496949999686, "nonpadding_tokens": 9119, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21272554400002264, "nonpadding_tokens": 4631, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035788899992894585, "prefetch": {"data_wait_seconds": 0.019010311001082414, "consumed_batches": 2019, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2569, "tokens": 188489678, "loss_tokens": 107522, "loss": 2.1392212175182754, "lr": 0.0002992381271109239, "optimizer_lrs": [0.019949208474061595, 0.0002992381271109239], "gradient_norm": 0.7939000129699707, "seconds": 0.33167690300001595, "nonpadding_tokens_per_second": 324176.9294981473, "ranks": [{"compute_seconds": 0.32897282000010364, "nonpadding_tokens": 15908, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32935944700000164, "nonpadding_tokens": 3323, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32618071899992174, "nonpadding_tokens": 5567, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32871249499999067, "nonpadding_tokens": 19767, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32933147400001417, "nonpadding_tokens": 24376, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3281819929999301, "nonpadding_tokens": 12713, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32860328300000674, "nonpadding_tokens": 14581, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32560616000000664, "nonpadding_tokens": 11287, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039157599996997305, "prefetch": {"data_wait_seconds": 0.019017958001157353, "consumed_batches": 2020, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2570, "tokens": 188543154, "loss_tokens": 53476, "loss": 2.137575442721969, "lr": 0.000299237643876751, "optimizer_lrs": [0.01994917625845007, 0.000299237643876751], "gradient_norm": 0.5301557779312134, "seconds": 0.22016235299997788, "nonpadding_tokens_per_second": 242893.47961322602, "ranks": [{"compute_seconds": 0.21713046399997893, "nonpadding_tokens": 2478, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21768314599989935, "nonpadding_tokens": 12283, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21501614099997823, "nonpadding_tokens": 2272, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21563456500007305, "nonpadding_tokens": 10520, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21421959099996002, "nonpadding_tokens": 5129, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2153329020000001, "nonpadding_tokens": 12371, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21424535200003447, "nonpadding_tokens": 2689, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21451884399994015, "nonpadding_tokens": 5734, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003790089999711199, "prefetch": {"data_wait_seconds": 0.019026218001229722, "consumed_batches": 2021, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2571, "tokens": 188620286, "loss_tokens": 77132, "loss": 2.506977081496655, "lr": 0.00029923694660680166, "optimizer_lrs": [0.019949129773786777, 0.00029923694660680166], "gradient_norm": 0.44567522406578064, "seconds": 0.3059133990000191, "nonpadding_tokens_per_second": 252136.7166398461, "ranks": [{"compute_seconds": 0.2984009240000205, "nonpadding_tokens": 5950, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2983430379999845, "nonpadding_tokens": 13080, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2969877240000187, "nonpadding_tokens": 9366, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29775249700003315, "nonpadding_tokens": 7608, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30248517799998353, "nonpadding_tokens": 2828, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2991749870000149, "nonpadding_tokens": 7759, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2952639339999905, "nonpadding_tokens": 18925, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3033746940000128, "nonpadding_tokens": 11616, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000385081999979775, "prefetch": {"data_wait_seconds": 0.019033695001169804, "consumed_batches": 2022, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2572, "tokens": 188665750, "loss_tokens": 45464, "loss": 2.4943605792495163, "lr": 0.0002992365354653279, "optimizer_lrs": [0.019949102364355195, 0.0002992365354653279], "gradient_norm": 0.45216476917266846, "seconds": 0.3640857190000588, "nonpadding_tokens_per_second": 124871.69264662275, "ranks": [{"compute_seconds": 0.3614826240000184, "nonpadding_tokens": 18015, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3582975010000382, "nonpadding_tokens": 2611, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36095131699994454, "nonpadding_tokens": 4447, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35822914699997455, "nonpadding_tokens": 4333, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3566291149999188, "nonpadding_tokens": 4986, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3593925950000312, "nonpadding_tokens": 1986, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3615623919999962, "nonpadding_tokens": 1776, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35747174699997686, "nonpadding_tokens": 7310, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047292099998230697, "prefetch": {"data_wait_seconds": 0.019043465001189475, "consumed_batches": 2023, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2573, "tokens": 188742788, "loss_tokens": 77038, "loss": 2.2834575144733766, "lr": 0.00029923583854079004, "optimizer_lrs": [0.019949055902719337, 0.00029923583854079004], "gradient_norm": 0.386300653219223, "seconds": 0.2715129350000325, "nonpadding_tokens_per_second": 283736.0216373882, "ranks": [{"compute_seconds": 0.26810594400001264, "nonpadding_tokens": 5414, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26869722300000376, "nonpadding_tokens": 10902, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.267820692999976, "nonpadding_tokens": 8305, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26624377200005256, "nonpadding_tokens": 7321, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2691081979999126, "nonpadding_tokens": 10285, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.267390837999983, "nonpadding_tokens": 16122, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26876396099999056, "nonpadding_tokens": 15367, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26524645599999985, "nonpadding_tokens": 3322, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039393499992002035, "prefetch": {"data_wait_seconds": 0.019053699001190694, "consumed_batches": 2024, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2574, "tokens": 188776905, "loss_tokens": 34117, "loss": 2.382624727042237, "lr": 0.00029923552979993647, "optimizer_lrs": [0.019949035319995766, 0.00029923552979993647], "gradient_norm": 0.4280458390712738, "seconds": 0.21291303700002118, "nonpadding_tokens_per_second": 160239.13087105422, "ranks": [{"compute_seconds": 0.21063680499992188, "nonpadding_tokens": 2035, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20615535400008866, "nonpadding_tokens": 3519, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20667402799995216, "nonpadding_tokens": 2344, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20736380200003168, "nonpadding_tokens": 5105, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20988091399999576, "nonpadding_tokens": 10576, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20726748100003078, "nonpadding_tokens": 2543, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20948247499995887, "nonpadding_tokens": 1592, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20646538300002248, "nonpadding_tokens": 6403, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036390399998253997, "prefetch": {"data_wait_seconds": 0.019063392001228863, "consumed_batches": 2025, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2575, "tokens": 188849322, "loss_tokens": 72417, "loss": 2.1388917657456124, "lr": 0.00029923487425821964, "optimizer_lrs": [0.019948991617214643, 0.00029923487425821964], "gradient_norm": 0.5864160060882568, "seconds": 0.354718489999982, "nonpadding_tokens_per_second": 204153.44009838245, "ranks": [{"compute_seconds": 0.35185902000000624, "nonpadding_tokens": 12629, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35135325200008083, "nonpadding_tokens": 3547, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35185246200001075, "nonpadding_tokens": 8889, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3522902359999307, "nonpadding_tokens": 4074, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3476313559999653, "nonpadding_tokens": 6059, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3506414120000727, "nonpadding_tokens": 3829, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3472566819999656, "nonpadding_tokens": 16762, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34799195800007965, "nonpadding_tokens": 16628, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035224800001287804, "prefetch": {"data_wait_seconds": 0.019071415001192804, "consumed_batches": 2026, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2576, "tokens": 188954793, "loss_tokens": 105471, "loss": 2.348286051379052, "lr": 0.0002992339190000454, "optimizer_lrs": [0.01994892793333636, 0.0002992339190000454], "gradient_norm": 0.4733276665210724, "seconds": 0.30792597000004207, "nonpadding_tokens_per_second": 342520.63897041744, "ranks": [{"compute_seconds": 0.2811027779999904, "nonpadding_tokens": 3294, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28208409199999096, "nonpadding_tokens": 4006, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2828166040000042, "nonpadding_tokens": 25374, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2849695489999249, "nonpadding_tokens": 22539, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2817278500000384, "nonpadding_tokens": 24540, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2827831659999447, "nonpadding_tokens": 7067, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28165494500001387, "nonpadding_tokens": 12359, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2821087959999886, "nonpadding_tokens": 6292, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.007354789000032724, "prefetch": {"data_wait_seconds": 0.019082255001151793, "consumed_batches": 2027, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2577, "tokens": 189010979, "loss_tokens": 56186, "loss": 2.1761240010856797, "lr": 0.0002992334098769369, "optimizer_lrs": [0.019948893991795794, 0.0002992334098769369], "gradient_norm": 0.45585888624191284, "seconds": 0.22549227599995447, "nonpadding_tokens_per_second": 249170.3973044795, "ranks": [{"compute_seconds": 0.2222637659999691, "nonpadding_tokens": 4491, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2225662440000633, "nonpadding_tokens": 11095, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22177908599996954, "nonpadding_tokens": 10291, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22009799699992527, "nonpadding_tokens": 11246, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21865686799992545, "nonpadding_tokens": 5767, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21968853700002455, "nonpadding_tokens": 6627, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21854888600000777, "nonpadding_tokens": 4550, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22321307500010334, "nonpadding_tokens": 2119, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034227599996938807, "prefetch": {"data_wait_seconds": 0.019091382001150237, "consumed_batches": 2028, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2578, "tokens": 189065630, "loss_tokens": 54651, "loss": 2.573872447439205, "lr": 0.0002992329145012669, "optimizer_lrs": [0.019948860966751126, 0.0002992329145012669], "gradient_norm": 0.4702033996582031, "seconds": 0.29150871399997413, "nonpadding_tokens_per_second": 187476.38535431516, "ranks": [{"compute_seconds": 0.28717775400002665, "nonpadding_tokens": 8913, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28917933300010645, "nonpadding_tokens": 5268, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2880496299999322, "nonpadding_tokens": 2303, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2869665239999222, "nonpadding_tokens": 12581, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28898105999996915, "nonpadding_tokens": 3313, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.286019916999976, "nonpadding_tokens": 8066, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28870666399996026, "nonpadding_tokens": 9619, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28541096000003563, "nonpadding_tokens": 4588, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004059750000351414, "prefetch": {"data_wait_seconds": 0.01910054900118041, "consumed_batches": 2029, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2579, "tokens": 189160602, "loss_tokens": 94972, "loss": 2.140586666333235, "lr": 0.0002992320532624851, "optimizer_lrs": [0.019948803550832343, 0.0002992320532624851], "gradient_norm": 0.5322856307029724, "seconds": 0.3041511270000683, "nonpadding_tokens_per_second": 312252.66510348546, "ranks": [{"compute_seconds": 0.29877635299999383, "nonpadding_tokens": 20651, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2983078130000649, "nonpadding_tokens": 21657, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30100589899996066, "nonpadding_tokens": 4660, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3018360959999882, "nonpadding_tokens": 5694, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29620870799999466, "nonpadding_tokens": 9757, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29897400600009405, "nonpadding_tokens": 10049, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3018649619999678, "nonpadding_tokens": 6667, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2972053800000367, "nonpadding_tokens": 15837, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003431510000382332, "prefetch": {"data_wait_seconds": 0.019108296001220992, "consumed_batches": 2030, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2580, "tokens": 189246106, "loss_tokens": 85504, "loss": 2.2499002239661303, "lr": 0.00029923127747061876, "optimizer_lrs": [0.019948751831374584, 0.00029923127747061876], "gradient_norm": 0.7196201682090759, "seconds": 0.3489761839999801, "nonpadding_tokens_per_second": 245013.85458442883, "ranks": [{"compute_seconds": 0.3463282599999502, "nonpadding_tokens": 3887, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34664093399999274, "nonpadding_tokens": 7807, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34511577599994325, "nonpadding_tokens": 12076, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34388783299993975, "nonpadding_tokens": 16712, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34220690899996953, "nonpadding_tokens": 17022, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3433258239999759, "nonpadding_tokens": 18055, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3424735390000251, "nonpadding_tokens": 7988, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34249720399998296, "nonpadding_tokens": 1957, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003432600000223829, "prefetch": {"data_wait_seconds": 0.019116808001243157, "consumed_batches": 2031, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2581, "tokens": 189287378, "loss_tokens": 41272, "loss": 2.5302342538524907, "lr": 0.00029923090286321734, "optimizer_lrs": [0.019948726857547824, 0.00029923090286321734], "gradient_norm": 0.5118117928504944, "seconds": 0.26020350499993583, "nonpadding_tokens_per_second": 158614.3122861092, "ranks": [{"compute_seconds": 0.25712180200002877, "nonpadding_tokens": 3476, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.257473363000031, "nonpadding_tokens": 3561, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2547735800000055, "nonpadding_tokens": 2614, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2570373669999526, "nonpadding_tokens": 7541, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25780451900004664, "nonpadding_tokens": 5049, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2563082570000006, "nonpadding_tokens": 8788, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25358648600001743, "nonpadding_tokens": 2669, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2543489580000369, "nonpadding_tokens": 7574, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040783499991903227, "prefetch": {"data_wait_seconds": 0.019126265001162324, "consumed_batches": 2032, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2582, "tokens": 189361983, "loss_tokens": 74605, "loss": 2.4187045523088266, "lr": 0.00029923022547624823, "optimizer_lrs": [0.01994868169841655, 0.00029923022547624823], "gradient_norm": 0.4815807640552521, "seconds": 0.3584431259999974, "nonpadding_tokens_per_second": 208136.2274471419, "ranks": [{"compute_seconds": 0.35561342899995907, "nonpadding_tokens": 18529, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3531106789999967, "nonpadding_tokens": 18476, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35307314600004247, "nonpadding_tokens": 9221, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3538292119999369, "nonpadding_tokens": 1782, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3520671950000178, "nonpadding_tokens": 3411, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35483480799996414, "nonpadding_tokens": 2740, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3561067150000099, "nonpadding_tokens": 11316, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3530008210000233, "nonpadding_tokens": 9130, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000372630000015306, "prefetch": {"data_wait_seconds": 0.019135677001102636, "consumed_batches": 2033, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2583, "tokens": 189407965, "loss_tokens": 45982, "loss": 2.259379859238398, "lr": 0.00029922980782788036, "optimizer_lrs": [0.019948653855192025, 0.00029922980782788036], "gradient_norm": 0.5009034276008606, "seconds": 0.2613068680000197, "nonpadding_tokens_per_second": 175969.3511002418, "ranks": [{"compute_seconds": 0.2585923469999898, "nonpadding_tokens": 4346, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25896261800005504, "nonpadding_tokens": 2870, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2579391400000759, "nonpadding_tokens": 1497, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25650105499994424, "nonpadding_tokens": 5823, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25526055299997097, "nonpadding_tokens": 8969, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25761401099998693, "nonpadding_tokens": 6563, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2550832570000239, "nonpadding_tokens": 11043, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25529524299997775, "nonpadding_tokens": 4871, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039829799993640336, "prefetch": {"data_wait_seconds": 0.019145098001175143, "consumed_batches": 2034, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2584, "tokens": 189521496, "loss_tokens": 113531, "loss": 2.7517803507412073, "lr": 0.0002992287761572694, "optimizer_lrs": [0.019948585077151294, 0.0002992287761572694], "gradient_norm": 0.4463741183280945, "seconds": 0.32100790899994536, "nonpadding_tokens_per_second": 353670.41377170343, "ranks": [{"compute_seconds": 0.3170393549999062, "nonpadding_tokens": 3908, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31860439199999746, "nonpadding_tokens": 20018, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3174745119999898, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31798070299998926, "nonpadding_tokens": 14455, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31445559100006903, "nonpadding_tokens": 14498, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3172902229999863, "nonpadding_tokens": 13021, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.318201920999968, "nonpadding_tokens": 9503, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3186575480000329, "nonpadding_tokens": 5360, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004322670000647122, "prefetch": {"data_wait_seconds": 0.01915411900115771, "consumed_batches": 2035, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2585, "tokens": 189573926, "loss_tokens": 52430, "loss": 2.1346348703032616, "lr": 0.0002992282994867991, "optimizer_lrs": [0.01994855329911994, 0.0002992282994867991], "gradient_norm": 0.5616271495819092, "seconds": 0.18844861900004162, "nonpadding_tokens_per_second": 278219.0725419347, "ranks": [{"compute_seconds": 0.17410790000008092, "nonpadding_tokens": 4787, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.17297966000000997, "nonpadding_tokens": 11130, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.16986599799997748, "nonpadding_tokens": 1380, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.17062899200004722, "nonpadding_tokens": 10888, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.17382577299997592, "nonpadding_tokens": 2892, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.17147530099998676, "nonpadding_tokens": 12946, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.1688882050000302, "nonpadding_tokens": 6072, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.17374695000000884, "nonpadding_tokens": 2335, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.010745237000037378, "prefetch": {"data_wait_seconds": 0.019162232001193047, "consumed_batches": 2036, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2586, "tokens": 189646881, "loss_tokens": 72955, "loss": 2.501216074977726, "lr": 0.0002992276359677497, "optimizer_lrs": [0.01994850906451665, 0.0002992276359677497], "gradient_norm": 1.6886191368103027, "seconds": 0.41724540500001694, "nonpadding_tokens_per_second": 174849.13944108514, "ranks": [{"compute_seconds": 0.41347487400003047, "nonpadding_tokens": 7733, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4148260700000037, "nonpadding_tokens": 4895, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4141918209999176, "nonpadding_tokens": 8631, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4124270790000537, "nonpadding_tokens": 6596, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4105089520001002, "nonpadding_tokens": 10867, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4119271000000708, "nonpadding_tokens": 20387, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4144944189999933, "nonpadding_tokens": 6647, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.41486929300003794, "nonpadding_tokens": 7199, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004193610000129411, "prefetch": {"data_wait_seconds": 0.019171490001170355, "consumed_batches": 2037, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2587, "tokens": 189704188, "loss_tokens": 57307, "loss": 2.591523232327639, "lr": 0.0002992271145664499, "optimizer_lrs": [0.019948474304429996, 0.0002992271145664499], "gradient_norm": 1.727669358253479, "seconds": 0.3105127449999827, "nonpadding_tokens_per_second": 184556.03166949938, "ranks": [{"compute_seconds": 0.3079293640000742, "nonpadding_tokens": 14133, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3047161240000378, "nonpadding_tokens": 3875, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3052264100000457, "nonpadding_tokens": 2682, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30602362100000846, "nonpadding_tokens": 5408, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3081813219999958, "nonpadding_tokens": 13579, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3053468499999781, "nonpadding_tokens": 5958, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30441742200002864, "nonpadding_tokens": 7151, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30469872300000134, "nonpadding_tokens": 4521, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003889810000146099, "prefetch": {"data_wait_seconds": 0.01918159000111075, "consumed_batches": 2038, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2588, "tokens": 189791903, "loss_tokens": 87715, "loss": 2.003985207775181, "lr": 0.0002992263161616296, "optimizer_lrs": [0.019948421077441974, 0.0002992263161616296], "gradient_norm": 0.5692802667617798, "seconds": 0.24430489199994554, "nonpadding_tokens_per_second": 359039.0650057861, "ranks": [{"compute_seconds": 0.2416261429999622, "nonpadding_tokens": 7485, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24183649899998727, "nonpadding_tokens": 9956, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23871276399995622, "nonpadding_tokens": 20571, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23927277099994626, "nonpadding_tokens": 8246, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24204763799991724, "nonpadding_tokens": 10444, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2389295159999847, "nonpadding_tokens": 13436, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2376819669999577, "nonpadding_tokens": 7648, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2384751800000231, "nonpadding_tokens": 9929, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000357441000005565, "prefetch": {"data_wait_seconds": 0.01919074600102988, "consumed_batches": 2039, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2589, "tokens": 189830618, "loss_tokens": 38715, "loss": 2.317039180550174, "lr": 0.0002992259636368654, "optimizer_lrs": [0.01994839757579103, 0.0002992259636368654], "gradient_norm": 0.4723697900772095, "seconds": 0.2644779949999929, "nonpadding_tokens_per_second": 146382.68866187162, "ranks": [{"compute_seconds": 0.26103901999999835, "nonpadding_tokens": 6486, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25799882599994817, "nonpadding_tokens": 10247, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25875486500001443, "nonpadding_tokens": 4447, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25795628499997747, "nonpadding_tokens": 1594, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2611520750000409, "nonpadding_tokens": 4742, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2580244360000279, "nonpadding_tokens": 2981, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2561009999999442, "nonpadding_tokens": 3970, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2620744159999049, "nonpadding_tokens": 4248, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004015849999632337, "prefetch": {"data_wait_seconds": 0.0192000560010257, "consumed_batches": 2040, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2590, "tokens": 189917302, "loss_tokens": 86684, "loss": 2.4442625297055973, "lr": 0.0002992251740334435, "optimizer_lrs": [0.0199483449355629, 0.0002992251740334435], "gradient_norm": 0.40194064378738403, "seconds": 0.36446145499996874, "nonpadding_tokens_per_second": 237841.33770746054, "ranks": [{"compute_seconds": 0.3618350869999176, "nonpadding_tokens": 17235, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3616509109999697, "nonpadding_tokens": 2614, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35875764100001106, "nonpadding_tokens": 5920, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3594936150000194, "nonpadding_tokens": 15815, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3620893419999902, "nonpadding_tokens": 7920, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3591040700000576, "nonpadding_tokens": 10280, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3578217179999683, "nonpadding_tokens": 17297, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35857820899991566, "nonpadding_tokens": 9603, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003771699999788325, "prefetch": {"data_wait_seconds": 0.019210663000947648, "consumed_batches": 2041, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2591, "tokens": 189973541, "loss_tokens": 56239, "loss": 2.400493874357652, "lr": 0.0002992246615384516, "optimizer_lrs": [0.019948310769230107, 0.0002992246615384516], "gradient_norm": 0.5038876533508301, "seconds": 0.36011731899998267, "nonpadding_tokens_per_second": 156168.55128259663, "ranks": [{"compute_seconds": 0.3573181869999189, "nonpadding_tokens": 11557, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.353328534999946, "nonpadding_tokens": 5696, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3539142480000237, "nonpadding_tokens": 6531, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35427150900000015, "nonpadding_tokens": 4119, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3527271689999907, "nonpadding_tokens": 11672, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35433452200004467, "nonpadding_tokens": 4873, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3571257800000467, "nonpadding_tokens": 2889, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35772015800000645, "nonpadding_tokens": 8902, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004451480000398078, "prefetch": {"data_wait_seconds": 0.019221842000959555, "consumed_batches": 2042, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2592, "tokens": 190026836, "loss_tokens": 53295, "loss": 2.2055350818557087, "lr": 0.00029922417571564327, "optimizer_lrs": [0.019948278381042885, 0.00029922417571564327], "gradient_norm": 0.44249212741851807, "seconds": 0.2699431349999486, "nonpadding_tokens_per_second": 197430.46993956764, "ranks": [{"compute_seconds": 0.2675889749999669, "nonpadding_tokens": 4468, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2641059250000808, "nonpadding_tokens": 2462, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2635024329999851, "nonpadding_tokens": 3177, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26687630399999307, "nonpadding_tokens": 6797, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26217512599998827, "nonpadding_tokens": 20832, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26460294299999987, "nonpadding_tokens": 3820, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2669817780000585, "nonpadding_tokens": 7339, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26758027400001083, "nonpadding_tokens": 4400, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003633439999930488, "prefetch": {"data_wait_seconds": 0.01923192400097662, "consumed_batches": 2043, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2593, "tokens": 190095528, "loss_tokens": 68692, "loss": 2.3951651939090435, "lr": 0.0002992235493141458, "optimizer_lrs": [0.019948236620943055, 0.0002992235493141458], "gradient_norm": 1.114583134651184, "seconds": 0.2810167789999696, "nonpadding_tokens_per_second": 244440.92002067761, "ranks": [{"compute_seconds": 0.19436356799997156, "nonpadding_tokens": 8039, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2110451019999573, "nonpadding_tokens": 2902, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2139425839999376, "nonpadding_tokens": 6330, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21228820600003928, "nonpadding_tokens": 25132, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21028219799995895, "nonpadding_tokens": 7732, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21146147399997517, "nonpadding_tokens": 5053, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21053320199996506, "nonpadding_tokens": 5431, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2145785640000213, "nonpadding_tokens": 8073, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.028635324999982004, "prefetch": {"data_wait_seconds": 0.019240807000940663, "consumed_batches": 2044, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2594, "tokens": 190197517, "loss_tokens": 101989, "loss": 2.3483157497377167, "lr": 0.00029922261881279796, "optimizer_lrs": [0.019948174587519865, 0.00029922261881279796], "gradient_norm": 0.5588428974151611, "seconds": 0.426700627999935, "nonpadding_tokens_per_second": 239017.69368854907, "ranks": [{"compute_seconds": 0.4235655460000771, "nonpadding_tokens": 17934, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.42052538099994763, "nonpadding_tokens": 24706, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.421393256999977, "nonpadding_tokens": 19754, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4212797010000031, "nonpadding_tokens": 2333, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4242566039999929, "nonpadding_tokens": 4597, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.42142949999993107, "nonpadding_tokens": 19233, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4237636059999659, "nonpadding_tokens": 11883, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.42431509699997605, "nonpadding_tokens": 1549, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004003639999154984, "prefetch": {"data_wait_seconds": 0.019250370000989392, "consumed_batches": 2045, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2595, "tokens": 190289637, "loss_tokens": 92120, "loss": 2.3543190946591404, "lr": 0.0002992217778741576, "optimizer_lrs": [0.019948118524943843, 0.0002992217778741576], "gradient_norm": 0.47987350821495056, "seconds": 0.2782172800000353, "nonpadding_tokens_per_second": 331108.11808665626, "ranks": [{"compute_seconds": 0.2742736740000282, "nonpadding_tokens": 10557, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27196919200002867, "nonpadding_tokens": 2738, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27284296799996355, "nonpadding_tokens": 27011, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27510490600002413, "nonpadding_tokens": 19127, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27589015299997754, "nonpadding_tokens": 15344, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2730973080000467, "nonpadding_tokens": 2705, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27537135800002943, "nonpadding_tokens": 11346, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27212302700002056, "nonpadding_tokens": 3292, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000403376000008393, "prefetch": {"data_wait_seconds": 0.019260428001032892, "consumed_batches": 2046, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2596, "tokens": 190337543, "loss_tokens": 47906, "loss": 2.3734222031165197, "lr": 0.0002992213403740445, "optimizer_lrs": [0.019948089358269635, 0.0002992213403740445], "gradient_norm": 0.469099223613739, "seconds": 0.2963684479999529, "nonpadding_tokens_per_second": 161643.38789535253, "ranks": [{"compute_seconds": 0.29308686599995326, "nonpadding_tokens": 5706, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2940277550000019, "nonpadding_tokens": 4574, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29073728200000915, "nonpadding_tokens": 4024, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2914375929999551, "nonpadding_tokens": 6358, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2940649669999402, "nonpadding_tokens": 3995, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2924396540000771, "nonpadding_tokens": 7419, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2897425150000572, "nonpadding_tokens": 10916, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29025473999990936, "nonpadding_tokens": 4914, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003444989999934478, "prefetch": {"data_wait_seconds": 0.019267956001002062, "consumed_batches": 2047, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2597, "tokens": 190385680, "loss_tokens": 48137, "loss": 2.4518525900035315, "lr": 0.00029922090064085627, "optimizer_lrs": [0.01994806004272375, 0.00029922090064085627], "gradient_norm": 0.4235917329788208, "seconds": 0.21233312900005785, "nonpadding_tokens_per_second": 226705.08472555352, "ranks": [{"compute_seconds": 0.21004782399995747, "nonpadding_tokens": 2224, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20677278300001944, "nonpadding_tokens": 9652, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20653808900010517, "nonpadding_tokens": 12231, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2071340820000387, "nonpadding_tokens": 4332, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20547679800006335, "nonpadding_tokens": 3065, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20767981899996357, "nonpadding_tokens": 13291, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2056073580000657, "nonpadding_tokens": 1785, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20635433500001454, "nonpadding_tokens": 1557, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035437399992588325, "prefetch": {"data_wait_seconds": 0.01927751400098998, "consumed_batches": 2048, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2598, "tokens": 190478343, "loss_tokens": 92663, "loss": 2.3176662340955936, "lr": 0.0002992200538126424, "optimizer_lrs": [0.019948003587509495, 0.0002992200538126424], "gradient_norm": 0.3607272803783417, "seconds": 0.4564168110000537, "nonpadding_tokens_per_second": 203022.75851094603, "ranks": [{"compute_seconds": 0.4520372849999603, "nonpadding_tokens": 17363, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4536156269999765, "nonpadding_tokens": 13473, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.452838163000024, "nonpadding_tokens": 10763, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4531956990000481, "nonpadding_tokens": 11807, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.449827821000099, "nonpadding_tokens": 16165, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.45088559300006636, "nonpadding_tokens": 3577, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4501046249999945, "nonpadding_tokens": 10741, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4540658920000169, "nonpadding_tokens": 8774, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038608199997725023, "prefetch": {"data_wait_seconds": 0.0192858570009804, "consumed_batches": 2049, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2599, "tokens": 190543033, "loss_tokens": 64690, "loss": 2.5188270501623125, "lr": 0.0002992194623520997, "optimizer_lrs": [0.019947964156806646, 0.0002992194623520997], "gradient_norm": 0.4093925356864929, "seconds": 0.23572273399997812, "nonpadding_tokens_per_second": 274432.58824584144, "ranks": [{"compute_seconds": 0.2310047729999951, "nonpadding_tokens": 5927, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23154195600000094, "nonpadding_tokens": 2916, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2285853829999951, "nonpadding_tokens": 3775, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22969938300002468, "nonpadding_tokens": 15592, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23303788699990946, "nonpadding_tokens": 3834, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23084261300004982, "nonpadding_tokens": 19600, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22791776399992614, "nonpadding_tokens": 8129, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23182791300007466, "nonpadding_tokens": 4917, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037321399997836124, "prefetch": {"data_wait_seconds": 0.019294795001044918, "consumed_batches": 2050, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2600, "tokens": 190606070, "loss_tokens": 63037, "loss": 2.5626023704332375, "lr": 0.0002992188857898999, "optimizer_lrs": [0.019947925719326663, 0.0002992188857898999], "gradient_norm": 0.41918841004371643, "seconds": 0.32414239000002, "nonpadding_tokens_per_second": 194473.17581633216, "ranks": [{"compute_seconds": 0.32097097099995153, "nonpadding_tokens": 9628, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31915955500005566, "nonpadding_tokens": 14363, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31649834199993165, "nonpadding_tokens": 2401, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3170032819999733, "nonpadding_tokens": 5717, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32127606699998523, "nonpadding_tokens": 2457, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31814842399990084, "nonpadding_tokens": 9963, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32151369000007435, "nonpadding_tokens": 6319, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.316203173999952, "nonpadding_tokens": 12189, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004330799999934243, "prefetch": {"data_wait_seconds": 0.019304875001083133, "consumed_batches": 2051, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2601, "tokens": 190695618, "loss_tokens": 89548, "loss": 2.683023525092688, "lr": 0.0002992180663823291, "optimizer_lrs": [0.019947871092155275, 0.0002992180663823291], "gradient_norm": 0.38278937339782715, "seconds": 0.2612600280000379, "nonpadding_tokens_per_second": 342754.3075972839, "ranks": [{"compute_seconds": 0.24512810299995635, "nonpadding_tokens": 8153, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25546985400001176, "nonpadding_tokens": 19920, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25518750900005216, "nonpadding_tokens": 9971, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25598369400006504, "nonpadding_tokens": 16312, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25858118799999374, "nonpadding_tokens": 7923, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25736763199995494, "nonpadding_tokens": 14548, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2584312010000076, "nonpadding_tokens": 4186, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25876268100000743, "nonpadding_tokens": 8535, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004335869999749775, "prefetch": {"data_wait_seconds": 0.019314660001171546, "consumed_batches": 2052, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2602, "tokens": 190784389, "loss_tokens": 88771, "loss": 2.244911591341767, "lr": 0.0002992172536619278, "optimizer_lrs": [0.019947816910795186, 0.0002992172536619278], "gradient_norm": 0.39309918880462646, "seconds": 0.297351492999951, "nonpadding_tokens_per_second": 298538.94158861553, "ranks": [{"compute_seconds": 0.29466693799997756, "nonpadding_tokens": 7475, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2910189570000057, "nonpadding_tokens": 2857, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2938243920000332, "nonpadding_tokens": 10313, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2918544980000206, "nonpadding_tokens": 13938, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29112909100001616, "nonpadding_tokens": 4783, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29326959200000147, "nonpadding_tokens": 16609, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29472838000003776, "nonpadding_tokens": 4399, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2912794620000341, "nonpadding_tokens": 28397, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000349203000041598, "prefetch": {"data_wait_seconds": 0.019324365001196497, "consumed_batches": 2053, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2603, "tokens": 190839484, "loss_tokens": 55095, "loss": 2.182892475496869, "lr": 0.0002992167490419279, "optimizer_lrs": [0.019947783269461863, 0.0002992167490419279], "gradient_norm": 0.4289289116859436, "seconds": 0.33559606400001485, "nonpadding_tokens_per_second": 164170.5785917607, "ranks": [{"compute_seconds": 0.33042606399999386, "nonpadding_tokens": 15200, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32978453499993066, "nonpadding_tokens": 2991, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33221048199993675, "nonpadding_tokens": 3032, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3302414739999904, "nonpadding_tokens": 2803, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32893895500001236, "nonpadding_tokens": 7992, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3304499679999253, "nonpadding_tokens": 3687, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3331721030000381, "nonpadding_tokens": 11378, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32938894899996285, "nonpadding_tokens": 8012, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004277330000377333, "prefetch": {"data_wait_seconds": 0.01934261800120396, "consumed_batches": 2054, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2604, "tokens": 190915150, "loss_tokens": 75666, "loss": 2.212958098749769, "lr": 0.00029921605574607273, "optimizer_lrs": [0.019947737049738182, 0.00029921605574607273], "gradient_norm": 0.5034943222999573, "seconds": 0.3188902819999839, "nonpadding_tokens_per_second": 237279.1027855901, "ranks": [{"compute_seconds": 0.3156575419999399, "nonpadding_tokens": 6226, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31598884699997143, "nonpadding_tokens": 13522, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31288075199995546, "nonpadding_tokens": 14583, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3155648529999553, "nonpadding_tokens": 6321, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3161157389999971, "nonpadding_tokens": 4370, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3152816100000564, "nonpadding_tokens": 9282, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31574031200000263, "nonpadding_tokens": 17521, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31661133399995833, "nonpadding_tokens": 3841, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033555799996065616, "prefetch": {"data_wait_seconds": 0.019351251001197056, "consumed_batches": 2055, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2605, "tokens": 190962564, "loss_tokens": 47414, "loss": 2.3849542065634624, "lr": 0.0002992156211556052, "optimizer_lrs": [0.01994770807704035, 0.0002992156211556052], "gradient_norm": 0.44859278202056885, "seconds": 0.368491027999994, "nonpadding_tokens_per_second": 128670.70402593566, "ranks": [{"compute_seconds": 0.36426128900006916, "nonpadding_tokens": 5151, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3628221079999321, "nonpadding_tokens": 4169, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36290680400009023, "nonpadding_tokens": 7213, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3630683210000143, "nonpadding_tokens": 6175, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36617078500000844, "nonpadding_tokens": 7085, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3637511560000348, "nonpadding_tokens": 4940, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36619927300000654, "nonpadding_tokens": 5709, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3620501670000067, "nonpadding_tokens": 6972, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036833100000421837, "prefetch": {"data_wait_seconds": 0.019358798001192667, "consumed_batches": 2056, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2606, "tokens": 191025181, "loss_tokens": 62617, "loss": 2.652326844147755, "lr": 0.0002992150470324445, "optimizer_lrs": [0.019947669802162966, 0.0002992150470324445], "gradient_norm": 0.4407329559326172, "seconds": 0.2950770380000449, "nonpadding_tokens_per_second": 212205.6003557636, "ranks": [{"compute_seconds": 0.29258571799994115, "nonpadding_tokens": 14688, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29276275800009444, "nonpadding_tokens": 13106, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2899776980000297, "nonpadding_tokens": 5476, "peak_allocated_bytes": 14997652992, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2902351789999784, "nonpadding_tokens": 7971, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2927325389999851, "nonpadding_tokens": 6651, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28973685199991905, "nonpadding_tokens": 7523, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2886151699999573, "nonpadding_tokens": 3623, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2888024439999981, "nonpadding_tokens": 3579, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033694900002956274, "prefetch": {"data_wait_seconds": 0.019366497001172434, "consumed_batches": 2057, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2607, "tokens": 191095468, "loss_tokens": 70287, "loss": 2.4713758411939617, "lr": 0.00029921440233506334, "optimizer_lrs": [0.01994762682233756, 0.00029921440233506334], "gradient_norm": 0.4419595003128052, "seconds": 0.23012426000002506, "nonpadding_tokens_per_second": 305430.6399507481, "ranks": [{"compute_seconds": 0.2258553499999607, "nonpadding_tokens": 2050, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22379466299992146, "nonpadding_tokens": 5322, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2245443049999949, "nonpadding_tokens": 16854, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2271520330000385, "nonpadding_tokens": 7370, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22780058099999678, "nonpadding_tokens": 15263, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22614450799994756, "nonpadding_tokens": 8407, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22323312100002113, "nonpadding_tokens": 4690, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2243337639999936, "nonpadding_tokens": 10331, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036983500001497305, "prefetch": {"data_wait_seconds": 0.019376342001194757, "consumed_batches": 2058, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2608, "tokens": 191177834, "loss_tokens": 82366, "loss": 2.24314947915402, "lr": 0.0002992136465090083, "optimizer_lrs": [0.01994757643393389, 0.0002992136465090083], "gradient_norm": 0.458334743976593, "seconds": 0.5183313660000977, "nonpadding_tokens_per_second": 158906.069365643, "ranks": [{"compute_seconds": 0.5159436629999163, "nonpadding_tokens": 18195, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5133892659999901, "nonpadding_tokens": 6829, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.5139989919999834, "nonpadding_tokens": 3021, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.5146343820000538, "nonpadding_tokens": 2208, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.512601946000018, "nonpadding_tokens": 10151, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5140025830000923, "nonpadding_tokens": 16412, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.5126171110000541, "nonpadding_tokens": 8107, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.513530338999999, "nonpadding_tokens": 17443, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037264900004174706, "prefetch": {"data_wait_seconds": 0.019385109001134992, "consumed_batches": 2059, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2609, "tokens": 191292301, "loss_tokens": 114467, "loss": 2.37481654101182, "lr": 0.0002992125955085797, "optimizer_lrs": [0.01994750636723865, 0.0002992125955085797], "gradient_norm": 0.4397304356098175, "seconds": 0.23308321900003648, "nonpadding_tokens_per_second": 491099.27557668614, "ranks": [{"compute_seconds": 0.2300577329999669, "nonpadding_tokens": 5599, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22685692699997162, "nonpadding_tokens": 5219, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22973960299998453, "nonpadding_tokens": 3547, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22805211699994743, "nonpadding_tokens": 18623, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22674867099999574, "nonpadding_tokens": 16396, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2277105880000363, "nonpadding_tokens": 25238, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2307348979999233, "nonpadding_tokens": 19843, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.226958513999989, "nonpadding_tokens": 20002, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035479199993915245, "prefetch": {"data_wait_seconds": 0.0193932890011439, "consumed_batches": 2060, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2610, "tokens": 191371803, "loss_tokens": 79502, "loss": 2.2319986839953714, "lr": 0.0002992118651340901, "optimizer_lrs": [0.019947457675606007, 0.0002992118651340901], "gradient_norm": 0.43771055340766907, "seconds": 0.27917354800001704, "nonpadding_tokens_per_second": 284776.26397467696, "ranks": [{"compute_seconds": 0.2744589229999974, "nonpadding_tokens": 2185, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.276277317999984, "nonpadding_tokens": 6928, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2754320279999547, "nonpadding_tokens": 23882, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27584572600005686, "nonpadding_tokens": 8498, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2722105020000072, "nonpadding_tokens": 3077, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2754504289999886, "nonpadding_tokens": 18372, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27613192399996933, "nonpadding_tokens": 14282, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2768564980000292, "nonpadding_tokens": 2278, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003519250000181273, "prefetch": {"data_wait_seconds": 0.01940249400111327, "consumed_batches": 2061, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2611, "tokens": 191410552, "loss_tokens": 38749, "loss": 2.4828860502851686, "lr": 0.0002992115090297204, "optimizer_lrs": [0.019947433935314694, 0.0002992115090297204], "gradient_norm": 0.44681084156036377, "seconds": 0.24140937300001042, "nonpadding_tokens_per_second": 160511.58046791467, "ranks": [{"compute_seconds": 0.16638396399991962, "nonpadding_tokens": 5906, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.1770947459999661, "nonpadding_tokens": 6462, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.17392996699993546, "nonpadding_tokens": 7055, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.1770764519999375, "nonpadding_tokens": 2341, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1728588140000511, "nonpadding_tokens": 6104, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1746352919999481, "nonpadding_tokens": 3232, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.1732560599999715, "nonpadding_tokens": 5582, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.17339150500004052, "nonpadding_tokens": 2067, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.017530177000026015, "prefetch": {"data_wait_seconds": 0.01940984300104276, "consumed_batches": 2062, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2612, "tokens": 191565104, "loss_tokens": 154552, "loss": 2.517133796392153, "lr": 0.0002992100878947876, "optimizer_lrs": [0.01994733919298584, 0.0002992100878947876], "gradient_norm": 0.4139106869697571, "seconds": 0.33246188900000107, "nonpadding_tokens_per_second": 464871.32845473156, "ranks": [{"compute_seconds": 0.3293203959999573, "nonpadding_tokens": 10705, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3297766299999694, "nonpadding_tokens": 13527, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32878447599989613, "nonpadding_tokens": 11638, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3278818720000345, "nonpadding_tokens": 25347, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3298337479999418, "nonpadding_tokens": 26438, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32693951099997776, "nonpadding_tokens": 25368, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3256678430000193, "nonpadding_tokens": 17875, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33015166300003784, "nonpadding_tokens": 23654, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038175600002432475, "prefetch": {"data_wait_seconds": 0.019418381001017337, "consumed_batches": 2063, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2613, "tokens": 191669802, "loss_tokens": 104698, "loss": 2.3606897935013085, "lr": 0.00029920912445197416, "optimizer_lrs": [0.019947274963464944, 0.00029920912445197416], "gradient_norm": 0.4025586247444153, "seconds": 0.33699743599993326, "nonpadding_tokens_per_second": 310678.92160467576, "ranks": [{"compute_seconds": 0.3346812399998953, "nonpadding_tokens": 10492, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3344757910000453, "nonpadding_tokens": 25044, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33372612700009086, "nonpadding_tokens": 12135, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33244146499998806, "nonpadding_tokens": 12858, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3347142669999812, "nonpadding_tokens": 25206, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33320933000004516, "nonpadding_tokens": 12527, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33036765899998954, "nonpadding_tokens": 3254, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3314132199999449, "nonpadding_tokens": 3182, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000348431000020355, "prefetch": {"data_wait_seconds": 0.01942772800096293, "consumed_batches": 2064, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2614, "tokens": 191723717, "loss_tokens": 53915, "loss": 2.499354307706575, "lr": 0.0002992086280916872, "optimizer_lrs": [0.01994724187277915, 0.0002992086280916872], "gradient_norm": 0.8963979482650757, "seconds": 0.272567247999973, "nonpadding_tokens_per_second": 197804.39651357284, "ranks": [{"compute_seconds": 0.26745531999995364, "nonpadding_tokens": 2104, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26896765300000425, "nonpadding_tokens": 2870, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2698061889999508, "nonpadding_tokens": 4946, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2701561620000348, "nonpadding_tokens": 12718, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2648400449999144, "nonpadding_tokens": 5204, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26791952600001423, "nonpadding_tokens": 4720, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2701926130000629, "nonpadding_tokens": 14235, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2646995659999902, "nonpadding_tokens": 7118, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035215999992033176, "prefetch": {"data_wait_seconds": 0.019436121000993012, "consumed_batches": 2065, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2615, "tokens": 191772787, "loss_tokens": 49070, "loss": 2.0383425463623395, "lr": 0.00029920817620120665, "optimizer_lrs": [0.01994721174674711, 0.00029920817620120665], "gradient_norm": 0.5682170391082764, "seconds": 0.2824739120000004, "nonpadding_tokens_per_second": 173715.15710095002, "ranks": [{"compute_seconds": 0.2800285859999576, "nonpadding_tokens": 2116, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27958602499995777, "nonpadding_tokens": 3425, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2784934129999783, "nonpadding_tokens": 14078, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2772418639999614, "nonpadding_tokens": 3651, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2752657439999666, "nonpadding_tokens": 3402, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2783449229999633, "nonpadding_tokens": 11643, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27529954600004203, "nonpadding_tokens": 7221, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2763837600000443, "nonpadding_tokens": 3534, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041671400003906456, "prefetch": {"data_wait_seconds": 0.019445046000967636, "consumed_batches": 2066, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2616, "tokens": 191865977, "loss_tokens": 93190, "loss": 2.5902918432771758, "lr": 0.0002992073176512535, "optimizer_lrs": [0.019947154510083567, 0.0002992073176512535], "gradient_norm": 0.4293175935745239, "seconds": 0.3085745079999924, "nonpadding_tokens_per_second": 302001.61576536414, "ranks": [{"compute_seconds": 0.3046974500000488, "nonpadding_tokens": 4088, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3063513239999338, "nonpadding_tokens": 18446, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3035740510000551, "nonpadding_tokens": 6201, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3057255260000602, "nonpadding_tokens": 11563, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3020442690000209, "nonpadding_tokens": 20718, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3033349799999314, "nonpadding_tokens": 7499, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30228816400006053, "nonpadding_tokens": 19937, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3031059229999755, "nonpadding_tokens": 4738, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036382500002218876, "prefetch": {"data_wait_seconds": 0.019453599001010957, "consumed_batches": 2067, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2617, "tokens": 191930612, "loss_tokens": 64635, "loss": 2.3962638759959773, "lr": 0.00029920672190318425, "optimizer_lrs": [0.019947114793545617, 0.00029920672190318425], "gradient_norm": 0.7309514284133911, "seconds": 0.2906885779999584, "nonpadding_tokens_per_second": 222351.357747566, "ranks": [{"compute_seconds": 0.28821430300001794, "nonpadding_tokens": 9672, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28788176900002327, "nonpadding_tokens": 6579, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2850357529999883, "nonpadding_tokens": 2577, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2853067020000708, "nonpadding_tokens": 9613, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2840400900000759, "nonpadding_tokens": 6448, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28497058400000697, "nonpadding_tokens": 3198, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2878294309999774, "nonpadding_tokens": 11568, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2884304379999776, "nonpadding_tokens": 14980, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003419759999587768, "prefetch": {"data_wait_seconds": 0.019461772000980204, "consumed_batches": 2068, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2618, "tokens": 191980260, "loss_tokens": 49648, "loss": 2.446174033949001, "lr": 0.000299206264140461, "optimizer_lrs": [0.019947084276030737, 0.000299206264140461], "gradient_norm": 0.4793684780597687, "seconds": 0.2728763640000125, "nonpadding_tokens_per_second": 181943.20413913802, "ranks": [{"compute_seconds": 0.26833038600000236, "nonpadding_tokens": 4418, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26720041099997616, "nonpadding_tokens": 3287, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2698524879999695, "nonpadding_tokens": 3175, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2678938100000323, "nonpadding_tokens": 3018, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2705912989999888, "nonpadding_tokens": 11590, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26788388099998883, "nonpadding_tokens": 11081, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26632736900000964, "nonpadding_tokens": 3823, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26689307000003737, "nonpadding_tokens": 9256, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003579379999791854, "prefetch": {"data_wait_seconds": 0.0194703020009257, "consumed_batches": 2069, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2619, "tokens": 192070620, "loss_tokens": 90360, "loss": 2.2658568849601592, "lr": 0.0002992054306685627, "optimizer_lrs": [0.019947028711237515, 0.0002992054306685627], "gradient_norm": 0.5734531879425049, "seconds": 0.3262002300000404, "nonpadding_tokens_per_second": 277007.77525505977, "ranks": [{"compute_seconds": 0.3239397080000117, "nonpadding_tokens": 19211, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32040183200001593, "nonpadding_tokens": 24414, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32085796199999095, "nonpadding_tokens": 10214, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3237950960000262, "nonpadding_tokens": 10932, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32009705400002986, "nonpadding_tokens": 7923, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32287913799996204, "nonpadding_tokens": 12008, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3198817990000862, "nonpadding_tokens": 4658, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32061911300002066, "nonpadding_tokens": 1000, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003385520000165343, "prefetch": {"data_wait_seconds": 0.019478963000963745, "consumed_batches": 2070, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2620, "tokens": 192146220, "loss_tokens": 75600, "loss": 2.701356853505291, "lr": 0.0002992047330064289, "optimizer_lrs": [0.019946982200428596, 0.0002992047330064289], "gradient_norm": 0.46921807527542114, "seconds": 0.2580519660000391, "nonpadding_tokens_per_second": 292964.2473639923, "ranks": [{"compute_seconds": 0.25426761699998224, "nonpadding_tokens": 5110, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25576645700004974, "nonpadding_tokens": 2714, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2537899419999121, "nonpadding_tokens": 4787, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25401569500002097, "nonpadding_tokens": 9616, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2519212180000068, "nonpadding_tokens": 18185, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2517128370000137, "nonpadding_tokens": 23972, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25426184200000534, "nonpadding_tokens": 5722, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2547572380000247, "nonpadding_tokens": 5494, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037348700004713464, "prefetch": {"data_wait_seconds": 0.01948862300100984, "consumed_batches": 2071, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2621, "tokens": 192204070, "loss_tokens": 57850, "loss": 2.313531493085566, "lr": 0.0002992041989410848, "optimizer_lrs": [0.019946946596072324, 0.0002992041989410848], "gradient_norm": 0.5806950330734253, "seconds": 0.27851127899998573, "nonpadding_tokens_per_second": 207711.51605678047, "ranks": [{"compute_seconds": 0.27540939099992556, "nonpadding_tokens": 4212, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27534875399999237, "nonpadding_tokens": 4246, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2721130700000458, "nonpadding_tokens": 4881, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27510669000002963, "nonpadding_tokens": 21085, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2761250259999315, "nonpadding_tokens": 2918, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27280222800004594, "nonpadding_tokens": 8419, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27135175199998685, "nonpadding_tokens": 7594, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2717820700000857, "nonpadding_tokens": 4495, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039349799999399693, "prefetch": {"data_wait_seconds": 0.019501032001087992, "consumed_batches": 2072, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2622, "tokens": 192268785, "loss_tokens": 64715, "loss": 2.017496909526385, "lr": 0.00029920360128694455, "optimizer_lrs": [0.01994690675246297, 0.00029920360128694455], "gradient_norm": 0.3701854944229126, "seconds": 0.2638920849999522, "nonpadding_tokens_per_second": 245232.819317077, "ranks": [{"compute_seconds": 0.26166600699991704, "nonpadding_tokens": 2465, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2571527810000589, "nonpadding_tokens": 8383, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25752007400001276, "nonpadding_tokens": 10094, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2584425160000592, "nonpadding_tokens": 5848, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2566437179999639, "nonpadding_tokens": 4163, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2586568909999869, "nonpadding_tokens": 18693, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26041657999996914, "nonpadding_tokens": 8202, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2572086710000576, "nonpadding_tokens": 6867, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003371929999502754, "prefetch": {"data_wait_seconds": 0.019510809001033635, "consumed_batches": 2073, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2623, "tokens": 192325526, "loss_tokens": 56741, "loss": 2.5695421410443946, "lr": 0.00029920307709003276, "optimizer_lrs": [0.019946871806002187, 0.00029920307709003276], "gradient_norm": 0.4407968521118164, "seconds": 0.31387690300005033, "nonpadding_tokens_per_second": 180774.69051614447, "ranks": [{"compute_seconds": 0.3114718010000388, "nonpadding_tokens": 4796, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3076804209999864, "nonpadding_tokens": 3095, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3105550300000459, "nonpadding_tokens": 15998, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30870243300000766, "nonpadding_tokens": 6171, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3069751369999949, "nonpadding_tokens": 6841, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31001834500000314, "nonpadding_tokens": 13276, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31064634199992724, "nonpadding_tokens": 5163, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3078683660000934, "nonpadding_tokens": 1401, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003727960000787789, "prefetch": {"data_wait_seconds": 0.01952004800109819, "consumed_batches": 2074, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2624, "tokens": 192370190, "loss_tokens": 44664, "loss": 2.3183747677100124, "lr": 0.00029920266434451455, "optimizer_lrs": [0.019946844289634304, 0.00029920266434451455], "gradient_norm": 0.4703046977519989, "seconds": 0.2810846599999195, "nonpadding_tokens_per_second": 158898.74602197355, "ranks": [{"compute_seconds": 0.27718446599999425, "nonpadding_tokens": 4802, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2747520739999345, "nonpadding_tokens": 3025, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27746817199999896, "nonpadding_tokens": 3417, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27872125999999753, "nonpadding_tokens": 4024, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2742619900000136, "nonpadding_tokens": 19332, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2773188940000182, "nonpadding_tokens": 2278, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.274443376000022, "nonpadding_tokens": 5924, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2752479549999407, "nonpadding_tokens": 1862, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038727100002233783, "prefetch": {"data_wait_seconds": 0.01952770400107795, "consumed_batches": 2075, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2625, "tokens": 192426849, "loss_tokens": 56659, "loss": 2.079053112700542, "lr": 0.00029920214059842335, "optimizer_lrs": [0.019946809373228225, 0.00029920214059842335], "gradient_norm": 0.42013972997665405, "seconds": 0.29373849399996743, "nonpadding_tokens_per_second": 192889.25747677553, "ranks": [{"compute_seconds": 0.2910583410000527, "nonpadding_tokens": 3471, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28625691300010203, "nonpadding_tokens": 10971, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2860733090000167, "nonpadding_tokens": 7512, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2867969500000527, "nonpadding_tokens": 10125, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29055349999998725, "nonpadding_tokens": 14460, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28748553600007654, "nonpadding_tokens": 6634, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28497877499989954, "nonpadding_tokens": 2084, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2853344530000186, "nonpadding_tokens": 1402, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003481309998960569, "prefetch": {"data_wait_seconds": 0.01953554600113705, "consumed_batches": 2076, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2626, "tokens": 192498386, "loss_tokens": 71537, "loss": 2.320069072647721, "lr": 0.000299201479077756, "optimizer_lrs": [0.0199467652718504, 0.000299201479077756], "gradient_norm": 0.4850316345691681, "seconds": 0.3553951380000626, "nonpadding_tokens_per_second": 201288.628771245, "ranks": [{"compute_seconds": 0.3527990789999649, "nonpadding_tokens": 14341, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3495897480000849, "nonpadding_tokens": 8023, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3498204680000754, "nonpadding_tokens": 5041, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3504356319999715, "nonpadding_tokens": 4756, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3530356919999349, "nonpadding_tokens": 5946, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35016216500002884, "nonpadding_tokens": 15462, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3527442390000033, "nonpadding_tokens": 14950, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34936062300005233, "nonpadding_tokens": 3018, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003596239999978934, "prefetch": {"data_wait_seconds": 0.019542892001098267, "consumed_batches": 2077, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2627, "tokens": 192565767, "loss_tokens": 67381, "loss": 2.2424399125866343, "lr": 0.0002992008557387115, "optimizer_lrs": [0.019946723715914102, 0.0002992008557387115], "gradient_norm": 0.4910939633846283, "seconds": 0.23897784299992963, "nonpadding_tokens_per_second": 281955.009527891, "ranks": [{"compute_seconds": 0.23613769799999318, "nonpadding_tokens": 7191, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.236192458000005, "nonpadding_tokens": 17655, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23538172400003532, "nonpadding_tokens": 6440, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23656090899999072, "nonpadding_tokens": 3468, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23186474199997065, "nonpadding_tokens": 12628, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23356316199999583, "nonpadding_tokens": 4134, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23611162200006675, "nonpadding_tokens": 9003, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23668895700006942, "nonpadding_tokens": 6862, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003791460000002189, "prefetch": {"data_wait_seconds": 0.019551733001165417, "consumed_batches": 2078, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2628, "tokens": 192643432, "loss_tokens": 77665, "loss": 2.623465766432756, "lr": 0.0002992001369619643, "optimizer_lrs": [0.019946675797464286, 0.0002992001369619643], "gradient_norm": 0.5238932967185974, "seconds": 0.38407023799993567, "nonpadding_tokens_per_second": 202215.616613368, "ranks": [{"compute_seconds": 0.3802709510000568, "nonpadding_tokens": 5862, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3805615470000703, "nonpadding_tokens": 2739, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.38027749399998356, "nonpadding_tokens": 10291, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3781225559999939, "nonpadding_tokens": 14428, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3763819019999346, "nonpadding_tokens": 7709, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3784665230000428, "nonpadding_tokens": 10958, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.38092823699992095, "nonpadding_tokens": 20059, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3815630819999569, "nonpadding_tokens": 5619, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033733999998730724, "prefetch": {"data_wait_seconds": 0.01955981400108442, "consumed_batches": 2079, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2629, "tokens": 192725975, "loss_tokens": 82543, "loss": 2.101641341482621, "lr": 0.0002991993726870078, "optimizer_lrs": [0.019946624845800523, 0.0002991993726870078], "gradient_norm": 0.49616244435310364, "seconds": 0.2931387150000546, "nonpadding_tokens_per_second": 281583.41350436985, "ranks": [{"compute_seconds": 0.28747912000005726, "nonpadding_tokens": 3941, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28941807100000005, "nonpadding_tokens": 8753, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28633164799998667, "nonpadding_tokens": 16602, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.286608096000009, "nonpadding_tokens": 6026, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2855524960000366, "nonpadding_tokens": 8240, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28810219699994377, "nonpadding_tokens": 16743, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2901148370000328, "nonpadding_tokens": 16998, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2908406640000294, "nonpadding_tokens": 5240, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036483700000644603, "prefetch": {"data_wait_seconds": 0.0195681810010484, "consumed_batches": 2080, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2630, "tokens": 192801219, "loss_tokens": 75244, "loss": 2.5636012090665035, "lr": 0.00029919867567728325, "optimizer_lrs": [0.01994657837848555, 0.00029919867567728325], "gradient_norm": 0.45815590023994446, "seconds": 0.32188098100004936, "nonpadding_tokens_per_second": 233763.4232573327, "ranks": [{"compute_seconds": 0.3193183409999847, "nonpadding_tokens": 10135, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.315658419999977, "nonpadding_tokens": 14535, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31653270800006794, "nonpadding_tokens": 11538, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3171092469999621, "nonpadding_tokens": 7003, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3152312540000821, "nonpadding_tokens": 2611, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31683777800003554, "nonpadding_tokens": 11202, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3190262389999816, "nonpadding_tokens": 8394, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.315862390999996, "nonpadding_tokens": 9826, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042131800000788644, "prefetch": {"data_wait_seconds": 0.019577427001038927, "consumed_batches": 2081, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2631, "tokens": 192847765, "loss_tokens": 46546, "loss": 2.3616534919756798, "lr": 0.00029919824435512216, "optimizer_lrs": [0.01994654962367481, 0.00029919824435512216], "gradient_norm": 0.41872814297676086, "seconds": 0.3046203430000105, "nonpadding_tokens_per_second": 152800.03804604211, "ranks": [{"compute_seconds": 0.2970532130000265, "nonpadding_tokens": 5294, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2978683280000496, "nonpadding_tokens": 5134, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30127801899993756, "nonpadding_tokens": 10387, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3020147280000174, "nonpadding_tokens": 1828, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29611850499998127, "nonpadding_tokens": 4750, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29919144699999833, "nonpadding_tokens": 6969, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3022611809999489, "nonpadding_tokens": 2978, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2969337600000017, "nonpadding_tokens": 9206, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003500199999280085, "prefetch": {"data_wait_seconds": 0.019586278001000323, "consumed_batches": 2082, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2632, "tokens": 192966870, "loss_tokens": 119105, "loss": 2.135624947525293, "lr": 0.0002991971401324516, "optimizer_lrs": [0.019946476008830108, 0.0002991971401324516], "gradient_norm": 0.5179149508476257, "seconds": 0.32893762499998047, "nonpadding_tokens_per_second": 362089.92510360305, "ranks": [{"compute_seconds": 0.32448963599995295, "nonpadding_tokens": 18481, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.326367045999973, "nonpadding_tokens": 18905, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3257003579999491, "nonpadding_tokens": 16979, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32659014500006833, "nonpadding_tokens": 18826, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32244725299995025, "nonpadding_tokens": 17573, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32391262600003756, "nonpadding_tokens": 18985, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3225085640000316, "nonpadding_tokens": 6210, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3232039749999558, "nonpadding_tokens": 3146, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003412999999454769, "prefetch": {"data_wait_seconds": 0.019594044001067346, "consumed_batches": 2083, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2633, "tokens": 193028915, "loss_tokens": 62045, "loss": 2.419834142557821, "lr": 0.0002991965646130208, "optimizer_lrs": [0.019946437640868054, 0.0002991965646130208], "gradient_norm": 0.3878525197505951, "seconds": 0.2612946770000235, "nonpadding_tokens_per_second": 237452.2156836529, "ranks": [{"compute_seconds": 0.25717905399994834, "nonpadding_tokens": 7945, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2590740429999414, "nonpadding_tokens": 8534, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2563198079999438, "nonpadding_tokens": 7002, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25684954500002277, "nonpadding_tokens": 5346, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.255140646999962, "nonpadding_tokens": 3537, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25797293799996623, "nonpadding_tokens": 15025, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25894755499996336, "nonpadding_tokens": 12128, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25541568500000267, "nonpadding_tokens": 2528, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000333379000039713, "prefetch": {"data_wait_seconds": 0.01960152900103651, "consumed_batches": 2084, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2634, "tokens": 193093515, "loss_tokens": 64600, "loss": 2.203800551470588, "lr": 0.0002991959651753608, "optimizer_lrs": [0.01994639767835739, 0.0002991959651753608], "gradient_norm": 0.3626527488231659, "seconds": 0.28587808000008863, "nonpadding_tokens_per_second": 225970.45565711078, "ranks": [{"compute_seconds": 0.2832625910000388, "nonpadding_tokens": 8265, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2835773969999309, "nonpadding_tokens": 10535, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2807532970000466, "nonpadding_tokens": 3885, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28350546399997256, "nonpadding_tokens": 5524, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2799415950000821, "nonpadding_tokens": 1125, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28090728900008344, "nonpadding_tokens": 9090, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2799127130000443, "nonpadding_tokens": 13573, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28029468000011093, "nonpadding_tokens": 12603, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003816220000771864, "prefetch": {"data_wait_seconds": 0.019611167001087892, "consumed_batches": 2085, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2635, "tokens": 193168741, "loss_tokens": 75226, "loss": 2.170982688830989, "lr": 0.0002991952668558305, "optimizer_lrs": [0.019946351123722034, 0.0002991952668558305], "gradient_norm": 0.3898215889930725, "seconds": 0.36739160100000845, "nonpadding_tokens_per_second": 204756.99443112276, "ranks": [{"compute_seconds": 0.36291054299999814, "nonpadding_tokens": 20367, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3649817929999699, "nonpadding_tokens": 10271, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36410253400003967, "nonpadding_tokens": 3732, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3644248279999829, "nonpadding_tokens": 12010, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36501833800002714, "nonpadding_tokens": 3322, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36350279599992064, "nonpadding_tokens": 6023, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36079643799996575, "nonpadding_tokens": 8682, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3615009650000047, "nonpadding_tokens": 10819, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003950719999465946, "prefetch": {"data_wait_seconds": 0.019621074001136662, "consumed_batches": 2086, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2636, "tokens": 193223749, "loss_tokens": 55008, "loss": 2.2552463959787667, "lr": 0.0002991947560277801, "optimizer_lrs": [0.019946317068518675, 0.0002991947560277801], "gradient_norm": 0.4276033341884613, "seconds": 0.2020689940000011, "nonpadding_tokens_per_second": 272223.8524134964, "ranks": [{"compute_seconds": 0.19980718099998285, "nonpadding_tokens": 4746, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19907303400009368, "nonpadding_tokens": 6609, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.1962058020000086, "nonpadding_tokens": 2092, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.19712134600001718, "nonpadding_tokens": 3142, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.19916055800001686, "nonpadding_tokens": 4463, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19791820299997198, "nonpadding_tokens": 4519, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19503027799999018, "nonpadding_tokens": 17570, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.1956145240000069, "nonpadding_tokens": 11867, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037088400006268785, "prefetch": {"data_wait_seconds": 0.019629049001153476, "consumed_batches": 2087, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2637, "tokens": 193283338, "loss_tokens": 59589, "loss": 2.450303065582574, "lr": 0.0002991942024762432, "optimizer_lrs": [0.01994628016508288, 0.0002991942024762432], "gradient_norm": 0.43333160877227783, "seconds": 0.21832515400001284, "nonpadding_tokens_per_second": 272936.94248348725, "ranks": [{"compute_seconds": 0.18873365299998568, "nonpadding_tokens": 3995, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.18780020500003047, "nonpadding_tokens": 9201, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.18791695699997035, "nonpadding_tokens": 8615, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.18834076899997854, "nonpadding_tokens": 7599, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.18712089000007381, "nonpadding_tokens": 7618, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1884143369999265, "nonpadding_tokens": 6036, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.18680880400006572, "nonpadding_tokens": 9643, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.19135633600001256, "nonpadding_tokens": 6882, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.009976442000038332, "prefetch": {"data_wait_seconds": 0.019638638001154618, "consumed_batches": 2088, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2638, "tokens": 193349971, "loss_tokens": 66633, "loss": 2.161676881200006, "lr": 0.0002991935832649459, "optimizer_lrs": [0.01994623888432973, 0.0002991935832649459], "gradient_norm": 0.38593778014183044, "seconds": 0.27931139199995414, "nonpadding_tokens_per_second": 238561.69819242798, "ranks": [{"compute_seconds": 0.27582334199996694, "nonpadding_tokens": 10990, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2758355189999975, "nonpadding_tokens": 10039, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2763135879999936, "nonpadding_tokens": 12519, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27701236199993673, "nonpadding_tokens": 5417, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27159241700007897, "nonpadding_tokens": 6731, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2746094069999572, "nonpadding_tokens": 3185, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27142688000003545, "nonpadding_tokens": 5030, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27219426500005284, "nonpadding_tokens": 12722, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034787800007052283, "prefetch": {"data_wait_seconds": 0.019647482001076355, "consumed_batches": 2089, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2639, "tokens": 193444114, "loss_tokens": 94143, "loss": 2.063791751643776, "lr": 0.00029919270800284655, "optimizer_lrs": [0.019946180533523105, 0.00029919270800284655], "gradient_norm": 0.7789852023124695, "seconds": 0.33947251299991876, "nonpadding_tokens_per_second": 277321.42189674877, "ranks": [{"compute_seconds": 0.3354761169999847, "nonpadding_tokens": 10033, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.333370876999993, "nonpadding_tokens": 16168, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.332514878999973, "nonpadding_tokens": 6331, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3332615019999139, "nonpadding_tokens": 9687, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33697921899999983, "nonpadding_tokens": 16986, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3343132350000815, "nonpadding_tokens": 23433, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33710057600001164, "nonpadding_tokens": 5055, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33215190700002495, "nonpadding_tokens": 6450, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042848600003253523, "prefetch": {"data_wait_seconds": 0.019656787001125053, "consumed_batches": 2090, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2640, "tokens": 193481605, "loss_tokens": 37491, "loss": 2.290239032367768, "lr": 0.0002991923593114183, "optimizer_lrs": [0.019946157287427886, 0.0002991923593114183], "gradient_norm": 0.45943599939346313, "seconds": 0.26409865499999796, "nonpadding_tokens_per_second": 141958.3147820283, "ranks": [{"compute_seconds": 0.2609415550000449, "nonpadding_tokens": 2599, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2617582530000391, "nonpadding_tokens": 2421, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2610532129999683, "nonpadding_tokens": 11699, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25896497999997337, "nonpadding_tokens": 7837, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2577837199999067, "nonpadding_tokens": 3432, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26014990199996646, "nonpadding_tokens": 2860, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26160986699994737, "nonpadding_tokens": 2165, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2583059979999689, "nonpadding_tokens": 4478, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003414430000248103, "prefetch": {"data_wait_seconds": 0.019665351001165732, "consumed_batches": 2091, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2641, "tokens": 193526106, "loss_tokens": 44501, "loss": 2.3441697588256445, "lr": 0.00029919194532486437, "optimizer_lrs": [0.019946129688324293, 0.00029919194532486437], "gradient_norm": 0.41631990671157837, "seconds": 0.26970966499993665, "nonpadding_tokens_per_second": 164995.94109840467, "ranks": [{"compute_seconds": 0.2666825119999885, "nonpadding_tokens": 2338, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2674262399999634, "nonpadding_tokens": 6764, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26408314699995117, "nonpadding_tokens": 2650, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26445040800001607, "nonpadding_tokens": 3396, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26301333499998236, "nonpadding_tokens": 9515, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2648562959999481, "nonpadding_tokens": 3460, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2672184320000497, "nonpadding_tokens": 8169, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.263697709999974, "nonpadding_tokens": 8209, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035864200003743463, "prefetch": {"data_wait_seconds": 0.01967483300127526, "consumed_batches": 2092, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2642, "tokens": 193582601, "loss_tokens": 56495, "loss": 2.2827504093282593, "lr": 0.00029919141960745954, "optimizer_lrs": [0.019946094640497306, 0.00029919141960745954], "gradient_norm": 0.4365043640136719, "seconds": 0.2650835780000307, "nonpadding_tokens_per_second": 213121.4631484771, "ranks": [{"compute_seconds": 0.2626704100000552, "nonpadding_tokens": 6301, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25969471799999155, "nonpadding_tokens": 6281, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2617643749999843, "nonpadding_tokens": 14059, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2603451900000664, "nonpadding_tokens": 3046, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25891980699998385, "nonpadding_tokens": 3619, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26159817000007024, "nonpadding_tokens": 6095, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25862599599997793, "nonpadding_tokens": 4200, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25941562399998475, "nonpadding_tokens": 12894, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003939040000204841, "prefetch": {"data_wait_seconds": 0.019684425001244676, "consumed_batches": 2093, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2643, "tokens": 193666589, "loss_tokens": 83988, "loss": 2.4320362358313092, "lr": 0.00029919063773734667, "optimizer_lrs": [0.019946042515823114, 0.00029919063773734667], "gradient_norm": 0.5010353922843933, "seconds": 0.3228555040000174, "nonpadding_tokens_per_second": 260141.14351290563, "ranks": [{"compute_seconds": 0.31352455199998985, "nonpadding_tokens": 19645, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3170748290000347, "nonpadding_tokens": 8359, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3203827209999872, "nonpadding_tokens": 2462, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3203506009999728, "nonpadding_tokens": 6929, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31249010299995916, "nonpadding_tokens": 30877, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31544589999998607, "nonpadding_tokens": 3659, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3203277740000203, "nonpadding_tokens": 5000, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31310847699990063, "nonpadding_tokens": 7057, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040573900002982555, "prefetch": {"data_wait_seconds": 0.019693641001211404, "consumed_batches": 2094, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2644, "tokens": 193720261, "loss_tokens": 53672, "loss": 2.5222855376173796, "lr": 0.0002991901378909152, "optimizer_lrs": [0.01994600919272768, 0.0002991901378909152], "gradient_norm": 0.4483470618724823, "seconds": 0.27547572200001014, "nonpadding_tokens_per_second": 194833.86633976415, "ranks": [{"compute_seconds": 0.2726791420000154, "nonpadding_tokens": 6030, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2689481090000072, "nonpadding_tokens": 2947, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26594993200001227, "nonpadding_tokens": 17581, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26692062399990846, "nonpadding_tokens": 1453, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2719100829999661, "nonpadding_tokens": 4446, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2675515749999704, "nonpadding_tokens": 9593, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2713284079999312, "nonpadding_tokens": 7976, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27312771000003977, "nonpadding_tokens": 3646, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003694260000202121, "prefetch": {"data_wait_seconds": 0.019702698001154317, "consumed_batches": 2095, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2645, "tokens": 193798468, "loss_tokens": 78207, "loss": 2.471687117201785, "lr": 0.00029918940927504945, "optimizer_lrs": [0.01994596061833663, 0.00029918940927504945], "gradient_norm": 0.7010671496391296, "seconds": 0.320542893000038, "nonpadding_tokens_per_second": 243982.94801685316, "ranks": [{"compute_seconds": 0.2561171549999699, "nonpadding_tokens": 2429, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2543003590000126, "nonpadding_tokens": 2833, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25138163099995836, "nonpadding_tokens": 4364, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25696917599998415, "nonpadding_tokens": 3608, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2576203879999639, "nonpadding_tokens": 13201, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2529191979999723, "nonpadding_tokens": 16290, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25020692000009603, "nonpadding_tokens": 20556, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2507026060000044, "nonpadding_tokens": 14926, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.04929278700001305, "prefetch": {"data_wait_seconds": 0.019712607001181937, "consumed_batches": 2096, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2646, "tokens": 193842566, "loss_tokens": 44098, "loss": 2.569067403850515, "lr": 0.00029918899829182495, "optimizer_lrs": [0.019945933219455, 0.00029918899829182495], "gradient_norm": 0.518244206905365, "seconds": 0.23639135300004455, "nonpadding_tokens_per_second": 186546.58658344278, "ranks": [{"compute_seconds": 0.23367431999997734, "nonpadding_tokens": 5314, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22947681300001932, "nonpadding_tokens": 13502, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23247627299997475, "nonpadding_tokens": 4501, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22980636800002685, "nonpadding_tokens": 8460, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22830849899992245, "nonpadding_tokens": 3706, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23054352600001948, "nonpadding_tokens": 4223, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2334910790000322, "nonpadding_tokens": 2024, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2340421139999762, "nonpadding_tokens": 2368, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039530499998363666, "prefetch": {"data_wait_seconds": 0.019721854001275574, "consumed_batches": 2097, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2647, "tokens": 193903227, "loss_tokens": 60661, "loss": 2.312048979162889, "lr": 0.000299188432775568, "optimizer_lrs": [0.019945895518371202, 0.000299188432775568], "gradient_norm": 0.4496251940727234, "seconds": 0.25261616199998116, "nonpadding_tokens_per_second": 240131.11243454218, "ranks": [{"compute_seconds": 0.24574901200003296, "nonpadding_tokens": 6753, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24657563899995694, "nonpadding_tokens": 2861, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24978508300000613, "nonpadding_tokens": 3277, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24942591700005323, "nonpadding_tokens": 8917, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2501416129999825, "nonpadding_tokens": 5249, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24568958099996507, "nonpadding_tokens": 3669, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2428313220000291, "nonpadding_tokens": 17939, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2431046309999374, "nonpadding_tokens": 11996, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034748099994885706, "prefetch": {"data_wait_seconds": 0.019729559001305574, "consumed_batches": 2098, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2648, "tokens": 193962325, "loss_tokens": 59098, "loss": 2.462784009188128, "lr": 0.00029918788164150754, "optimizer_lrs": [0.019945858776100504, 0.00029918788164150754], "gradient_norm": 0.569159746170044, "seconds": 0.34006712000007155, "nonpadding_tokens_per_second": 173783.34018292496, "ranks": [{"compute_seconds": 0.3350301920000902, "nonpadding_tokens": 10293, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3347421089999898, "nonpadding_tokens": 2428, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33143418899999233, "nonpadding_tokens": 4719, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3369112090000499, "nonpadding_tokens": 11374, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33740748599996095, "nonpadding_tokens": 7169, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3348060590000159, "nonpadding_tokens": 4033, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33032432300001346, "nonpadding_tokens": 13837, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3308042189999014, "nonpadding_tokens": 5245, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000338523999971585, "prefetch": {"data_wait_seconds": 0.01973707000134084, "consumed_batches": 2099, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2649, "tokens": 194042595, "loss_tokens": 80270, "loss": 2.084289935530086, "lr": 0.0002991871327636468, "optimizer_lrs": [0.019945808850909786, 0.0002991871327636468], "gradient_norm": 0.40452298521995544, "seconds": 0.288054279999983, "nonpadding_tokens_per_second": 278662.75758862094, "ranks": [{"compute_seconds": 0.2824319489999425, "nonpadding_tokens": 4055, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28144591599993873, "nonpadding_tokens": 18780, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2798995000000559, "nonpadding_tokens": 14415, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2845160399999713, "nonpadding_tokens": 17768, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27868983400003344, "nonpadding_tokens": 11964, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28156666599988966, "nonpadding_tokens": 905, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27842015599992465, "nonpadding_tokens": 2605, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28555853800003206, "nonpadding_tokens": 9778, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003924760000018068, "prefetch": {"data_wait_seconds": 0.01974709300129973, "consumed_batches": 2100, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2650, "tokens": 194100604, "loss_tokens": 58009, "loss": 2.3524754779430777, "lr": 0.0002991865913553018, "optimizer_lrs": [0.01994577275702012, 0.0002991865913553018], "gradient_norm": 0.48186829686164856, "seconds": 0.25003898400007074, "nonpadding_tokens_per_second": 231999.82287555444, "ranks": [{"compute_seconds": 0.24366118299997197, "nonpadding_tokens": 9305, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2457077259999778, "nonpadding_tokens": 9227, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2468021450000606, "nonpadding_tokens": 2024, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24368130700008805, "nonpadding_tokens": 1787, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24227082299989888, "nonpadding_tokens": 5673, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2453084050000598, "nonpadding_tokens": 3930, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2476727729999766, "nonpadding_tokens": 6616, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2425297419999879, "nonpadding_tokens": 19447, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044692200003737526, "prefetch": {"data_wait_seconds": 0.01975715300132208, "consumed_batches": 2101, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2651, "tokens": 194147845, "loss_tokens": 47241, "loss": 2.2555731514997563, "lr": 0.00029918615031386347, "optimizer_lrs": [0.019945743354257566, 0.00029918615031386347], "gradient_norm": 0.7042399048805237, "seconds": 0.26245588799997677, "nonpadding_tokens_per_second": 179995.96183570544, "ranks": [{"compute_seconds": 0.25827091600001495, "nonpadding_tokens": 5568, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2593501840000272, "nonpadding_tokens": 5570, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2590763089999655, "nonpadding_tokens": 6185, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2566263579999486, "nonpadding_tokens": 6627, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2549897700000656, "nonpadding_tokens": 2848, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2581770110000434, "nonpadding_tokens": 8567, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.259211677000053, "nonpadding_tokens": 7010, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2601115159999381, "nonpadding_tokens": 4866, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003953909999836469, "prefetch": {"data_wait_seconds": 0.01976614400128085, "consumed_batches": 2102, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2652, "tokens": 194218899, "loss_tokens": 71054, "loss": 2.2638193750527766, "lr": 0.000299185486730102, "optimizer_lrs": [0.019945699115340134, 0.000299185486730102], "gradient_norm": 0.4998144507408142, "seconds": 0.29066357399995013, "nonpadding_tokens_per_second": 244454.43583519754, "ranks": [{"compute_seconds": 0.28674847799993586, "nonpadding_tokens": 7892, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28504543299993657, "nonpadding_tokens": 9996, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2877372660000219, "nonpadding_tokens": 3521, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2883447149999938, "nonpadding_tokens": 14677, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2845081579999942, "nonpadding_tokens": 17111, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28592367700002796, "nonpadding_tokens": 4156, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2846287840000059, "nonpadding_tokens": 4203, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2850368959999514, "nonpadding_tokens": 9498, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033804899999267946, "prefetch": {"data_wait_seconds": 0.01977454600132944, "consumed_batches": 2103, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2653, "tokens": 194286654, "loss_tokens": 67755, "loss": 2.4581377758099032, "lr": 0.00029918485370507026, "optimizer_lrs": [0.019945656913671353, 0.00029918485370507026], "gradient_norm": 0.41768768429756165, "seconds": 0.2798497659999839, "nonpadding_tokens_per_second": 242112.04807655225, "ranks": [{"compute_seconds": 0.2776528330000474, "nonpadding_tokens": 8729, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27760125000008884, "nonpadding_tokens": 9885, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2769856240000763, "nonpadding_tokens": 8041, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27752210599999216, "nonpadding_tokens": 3814, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2743130869999959, "nonpadding_tokens": 5964, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27648048700007166, "nonpadding_tokens": 14290, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2735510239999712, "nonpadding_tokens": 9871, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2742147930000556, "nonpadding_tokens": 7161, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000351245000047129, "prefetch": {"data_wait_seconds": 0.01978264800136742, "consumed_batches": 2104, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2654, "tokens": 194331609, "loss_tokens": 44955, "loss": 2.4328215368145925, "lr": 0.0002991844335618092, "optimizer_lrs": [0.019945628904120615, 0.0002991844335618092], "gradient_norm": 0.4510662257671356, "seconds": 0.2520992309999883, "nonpadding_tokens_per_second": 178322.63835823475, "ranks": [{"compute_seconds": 0.2497966500000075, "nonpadding_tokens": 4300, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24917893899998944, "nonpadding_tokens": 5117, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24801790599997275, "nonpadding_tokens": 5383, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24875155000006544, "nonpadding_tokens": 6846, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24520934399993166, "nonpadding_tokens": 8033, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24799005900001703, "nonpadding_tokens": 8393, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24501019800004542, "nonpadding_tokens": 3144, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24566956999990452, "nonpadding_tokens": 3739, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004024239999580459, "prefetch": {"data_wait_seconds": 0.01979055300144239, "consumed_batches": 2105, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2655, "tokens": 194408006, "loss_tokens": 76397, "loss": 2.3881120168331216, "lr": 0.000299183719318396, "optimizer_lrs": [0.019945581287893067, 0.000299183719318396], "gradient_norm": 0.4621976315975189, "seconds": 0.3370399970000335, "nonpadding_tokens_per_second": 226670.42689296132, "ranks": [{"compute_seconds": 0.33433740899999975, "nonpadding_tokens": 12151, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33479497500002253, "nonpadding_tokens": 7051, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33214183600000524, "nonpadding_tokens": 9775, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3328531689999181, "nonpadding_tokens": 1833, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3313016489998972, "nonpadding_tokens": 5070, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3335516079999934, "nonpadding_tokens": 8691, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3347429100000454, "nonpadding_tokens": 13346, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3314733099999785, "nonpadding_tokens": 18480, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034606999997777166, "prefetch": {"data_wait_seconds": 0.01980010400139065, "consumed_batches": 2106, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2656, "tokens": 194476114, "loss_tokens": 68108, "loss": 2.1743981066835025, "lr": 0.0002991830823069168, "optimizer_lrs": [0.019945538820461118, 0.0002991830823069168], "gradient_norm": 0.5108819007873535, "seconds": 0.2564900419999958, "nonpadding_tokens_per_second": 265538.57400826935, "ranks": [{"compute_seconds": 0.2199484829999392, "nonpadding_tokens": 3341, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22522666200006825, "nonpadding_tokens": 4167, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22260461500002293, "nonpadding_tokens": 10703, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22337180900001385, "nonpadding_tokens": 7002, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22716183499994713, "nonpadding_tokens": 5788, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22411044200009655, "nonpadding_tokens": 21505, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22168945099997472, "nonpadding_tokens": 1798, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22220161899997493, "nonpadding_tokens": 13804, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.018349823999983528, "prefetch": {"data_wait_seconds": 0.01980894700143665, "consumed_batches": 2107, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2657, "tokens": 194535374, "loss_tokens": 59260, "loss": 2.2683183745359434, "lr": 0.0002991825278489121, "optimizer_lrs": [0.01994550185659414, 0.0002991825278489121], "gradient_norm": 0.4508524537086487, "seconds": 0.27835771799993836, "nonpadding_tokens_per_second": 212891.52830320704, "ranks": [{"compute_seconds": 0.275607998000055, "nonpadding_tokens": 6533, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2730000359999849, "nonpadding_tokens": 6725, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27116339700000935, "nonpadding_tokens": 10114, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27577105199998186, "nonpadding_tokens": 5099, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2699448809999012, "nonpadding_tokens": 5233, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2731720150000001, "nonpadding_tokens": 6515, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2760890130000462, "nonpadding_tokens": 15309, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27060214700009055, "nonpadding_tokens": 3732, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003511350000735547, "prefetch": {"data_wait_seconds": 0.019818423001495944, "consumed_batches": 2108, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2658, "tokens": 194590282, "loss_tokens": 54908, "loss": 2.378277499408101, "lr": 0.0002991820139424207, "optimizer_lrs": [0.01994546759616138, 0.0002991820139424207], "gradient_norm": 0.45977646112442017, "seconds": 0.2486158520000572, "nonpadding_tokens_per_second": 220854.78282369286, "ranks": [{"compute_seconds": 0.19255862099998922, "nonpadding_tokens": 7281, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.1988400760000104, "nonpadding_tokens": 4142, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2006785469999386, "nonpadding_tokens": 2645, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20176853700002084, "nonpadding_tokens": 7434, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.1999790639999901, "nonpadding_tokens": 12125, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20271114299998771, "nonpadding_tokens": 4712, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19997645099999772, "nonpadding_tokens": 12264, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20022650100008832, "nonpadding_tokens": 4305, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.007081373000005442, "prefetch": {"data_wait_seconds": 0.019828033001502376, "consumed_batches": 2109, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2659, "tokens": 194659109, "loss_tokens": 68827, "loss": 2.5272517507664145, "lr": 0.0002991813695349173, "optimizer_lrs": [0.019945424635661155, 0.0002991813695349173], "gradient_norm": 0.5772581696510315, "seconds": 0.28459804800002075, "nonpadding_tokens_per_second": 241839.32561615805, "ranks": [{"compute_seconds": 0.27953910999997333, "nonpadding_tokens": 13317, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27972678600008294, "nonpadding_tokens": 12236, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27657679399999324, "nonpadding_tokens": 6684, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2814135720000195, "nonpadding_tokens": 11533, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2821291300000439, "nonpadding_tokens": 6288, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27813055600006464, "nonpadding_tokens": 6441, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27554884000005586, "nonpadding_tokens": 4858, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2759804840000015, "nonpadding_tokens": 7470, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000385716999971919, "prefetch": {"data_wait_seconds": 0.019835442001479464, "consumed_batches": 2110, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2660, "tokens": 194701823, "loss_tokens": 42714, "loss": 2.332383703235473, "lr": 0.0002991809694887534, "optimizer_lrs": [0.019945397965916895, 0.0002991809694887534], "gradient_norm": 0.4484659731388092, "seconds": 0.23938624599998093, "nonpadding_tokens_per_second": 178431.30386030365, "ranks": [{"compute_seconds": 0.20552489099998184, "nonpadding_tokens": 4973, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2088601049999852, "nonpadding_tokens": 3285, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20572971400008555, "nonpadding_tokens": 8159, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20923456800005624, "nonpadding_tokens": 6628, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.20499519499992402, "nonpadding_tokens": 3299, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20790666400000646, "nonpadding_tokens": 10002, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20497487700004058, "nonpadding_tokens": 3944, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21015132599995923, "nonpadding_tokens": 2424, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01298915899997155, "prefetch": {"data_wait_seconds": 0.019848773001470477, "consumed_batches": 2111, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2661, "tokens": 194761969, "loss_tokens": 60146, "loss": 2.1844405176570345, "lr": 0.0002991804060146692, "optimizer_lrs": [0.01994536040097795, 0.0002991804060146692], "gradient_norm": 0.40943193435668945, "seconds": 0.35411060499995983, "nonpadding_tokens_per_second": 169850.8859965005, "ranks": [{"compute_seconds": 0.3511422029999949, "nonpadding_tokens": 7769, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3508558500000163, "nonpadding_tokens": 3023, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34802907899995716, "nonpadding_tokens": 2553, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3483373589999701, "nonpadding_tokens": 6993, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34675854300007813, "nonpadding_tokens": 15386, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3494555519999949, "nonpadding_tokens": 9219, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3467972420000933, "nonpadding_tokens": 14169, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35163110100006634, "nonpadding_tokens": 1034, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004239559999632547, "prefetch": {"data_wait_seconds": 0.019858005001381684, "consumed_batches": 2112, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2662, "tokens": 194823437, "loss_tokens": 61468, "loss": 2.250519579293291, "lr": 0.00029917982995592246, "optimizer_lrs": [0.019945321997061498, 0.00029917982995592246], "gradient_norm": 0.44453558325767517, "seconds": 0.4348219120000749, "nonpadding_tokens_per_second": 141363.6210679038, "ranks": [{"compute_seconds": 0.3945866689999775, "nonpadding_tokens": 3025, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4111934670000892, "nonpadding_tokens": 18804, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.41381223299993053, "nonpadding_tokens": 11251, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.41190404099995703, "nonpadding_tokens": 10706, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4108199690000447, "nonpadding_tokens": 3933, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4123687210000071, "nonpadding_tokens": 4834, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.41043562900006236, "nonpadding_tokens": 3295, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.41107905700005176, "nonpadding_tokens": 5620, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0165726000000177, "prefetch": {"data_wait_seconds": 0.01986682200147527, "consumed_batches": 2113, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2663, "tokens": 194891516, "loss_tokens": 68079, "loss": 2.522334576374506, "lr": 0.00029917919170549306, "optimizer_lrs": [0.019945279447032872, 0.00029917919170549306], "gradient_norm": 0.5399814248085022, "seconds": 0.2856983249999985, "nonpadding_tokens_per_second": 238289.8114645942, "ranks": [{"compute_seconds": 0.2806442760000891, "nonpadding_tokens": 1813, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2793749080000225, "nonpadding_tokens": 4075, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2820444480000788, "nonpadding_tokens": 10438, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.282674876000101, "nonpadding_tokens": 6740, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27876642799992624, "nonpadding_tokens": 9066, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28175617700003386, "nonpadding_tokens": 9921, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2788956629999575, "nonpadding_tokens": 15753, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28337786600002346, "nonpadding_tokens": 10273, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003665420000515951, "prefetch": {"data_wait_seconds": 0.019876757001455303, "consumed_batches": 2114, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2664, "tokens": 194932414, "loss_tokens": 40898, "loss": 2.3141907535209545, "lr": 0.0002991788081618568, "optimizer_lrs": [0.019945253877457124, 0.0002991788081618568], "gradient_norm": 0.43592369556427, "seconds": 0.20258921600009216, "nonpadding_tokens_per_second": 201876.4907998923, "ranks": [{"compute_seconds": 0.19859903200006102, "nonpadding_tokens": 3157, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2003257339999891, "nonpadding_tokens": 1810, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.19823293199999625, "nonpadding_tokens": 3586, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.19964312199999767, "nonpadding_tokens": 9450, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.19582071799993628, "nonpadding_tokens": 6225, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19919539300008182, "nonpadding_tokens": 1942, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.1958273799999688, "nonpadding_tokens": 6175, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.1966055629999346, "nonpadding_tokens": 8553, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000337112000011075, "prefetch": {"data_wait_seconds": 0.019885240001443094, "consumed_batches": 2115, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2665, "tokens": 195003713, "loss_tokens": 71299, "loss": 2.4218248152147996, "lr": 0.0002991781393024336, "optimizer_lrs": [0.01994520928682891, 0.0002991781393024336], "gradient_norm": 0.3850085437297821, "seconds": 0.27367038400007004, "nonpadding_tokens_per_second": 260528.73883489618, "ranks": [{"compute_seconds": 0.2708269360000486, "nonpadding_tokens": 1635, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2706417900000133, "nonpadding_tokens": 13090, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26769720400000097, "nonpadding_tokens": 2045, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2704155010000022, "nonpadding_tokens": 10183, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2669038730000466, "nonpadding_tokens": 5449, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26956713699996726, "nonpadding_tokens": 5183, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27140593400008584, "nonpadding_tokens": 19807, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2713368749999745, "nonpadding_tokens": 13907, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033772300002965494, "prefetch": {"data_wait_seconds": 0.019893149001475763, "consumed_batches": 2116, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2666, "tokens": 195083040, "loss_tokens": 79327, "loss": 2.50925323345141, "lr": 0.0002991773948129309, "optimizer_lrs": [0.019945159654195396, 0.0002991773948129309], "gradient_norm": 0.41113871335983276, "seconds": 0.3090201719999186, "nonpadding_tokens_per_second": 256704.92475171134, "ranks": [{"compute_seconds": 0.2987968569999566, "nonpadding_tokens": 2196, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3007781410000234, "nonpadding_tokens": 3264, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30493546799993965, "nonpadding_tokens": 2260, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29819595000003574, "nonpadding_tokens": 24728, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2970042070001, "nonpadding_tokens": 22185, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29936502600003223, "nonpadding_tokens": 11524, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30553051499998674, "nonpadding_tokens": 7052, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30656809699996757, "nonpadding_tokens": 6118, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003639700000803714, "prefetch": {"data_wait_seconds": 0.019902125001479476, "consumed_batches": 2117, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2667, "tokens": 195164951, "loss_tokens": 81911, "loss": 2.3495421097288522, "lr": 0.0002991766257197912, "optimizer_lrs": [0.019945108381319417, 0.0002991766257197912], "gradient_norm": 0.45055001974105835, "seconds": 0.41022824300000593, "nonpadding_tokens_per_second": 199671.77150208747, "ranks": [{"compute_seconds": 0.4066739720000214, "nonpadding_tokens": 2358, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.403066840000065, "nonpadding_tokens": 6589, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4056383680000408, "nonpadding_tokens": 3863, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4022650299999668, "nonpadding_tokens": 12642, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.40052591999995, "nonpadding_tokens": 6672, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4030782080000108, "nonpadding_tokens": 14618, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.40638431800005037, "nonpadding_tokens": 17664, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.40762814900006106, "nonpadding_tokens": 17505, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040135899996585067, "prefetch": {"data_wait_seconds": 0.019910982001491107, "consumed_batches": 2118, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2668, "tokens": 195228624, "loss_tokens": 63673, "loss": 2.301616511315628, "lr": 0.00029917602762259286, "optimizer_lrs": [0.019945068508172858, 0.00029917602762259286], "gradient_norm": 0.37505394220352173, "seconds": 0.3366371250000384, "nonpadding_tokens_per_second": 189144.3197181319, "ranks": [{"compute_seconds": 0.31924006799999916, "nonpadding_tokens": 12261, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33409015699999145, "nonpadding_tokens": 8250, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3333023540000113, "nonpadding_tokens": 8392, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3318253969999887, "nonpadding_tokens": 2048, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3300977450001028, "nonpadding_tokens": 3067, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33157593099997484, "nonpadding_tokens": 18262, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3337194119999367, "nonpadding_tokens": 2382, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3309913250000136, "nonpadding_tokens": 9011, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045673900001474976, "prefetch": {"data_wait_seconds": 0.019920310001410257, "consumed_batches": 2119, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2669, "tokens": 195284344, "loss_tokens": 55720, "loss": 1.874796835741206, "lr": 0.00029917550405239704, "optimizer_lrs": [0.01994503360349314, 0.00029917550405239704], "gradient_norm": 0.4424476623535156, "seconds": 0.23499575699997877, "nonpadding_tokens_per_second": 237110.66408745854, "ranks": [{"compute_seconds": 0.23069457400004012, "nonpadding_tokens": 5178, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22875160699993558, "nonpadding_tokens": 2885, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22656889899997168, "nonpadding_tokens": 13294, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22651299300002847, "nonpadding_tokens": 16643, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2326416490000156, "nonpadding_tokens": 3763, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22984263400007876, "nonpadding_tokens": 5234, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2253007819999766, "nonpadding_tokens": 5249, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.225889522999978, "nonpadding_tokens": 3474, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000357097000005524, "prefetch": {"data_wait_seconds": 0.01992802200136623, "consumed_batches": 2120, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2670, "tokens": 195360353, "loss_tokens": 76009, "loss": 2.441452451354445, "lr": 0.0002991747895703071, "optimizer_lrs": [0.01994498597135381, 0.0002991747895703071], "gradient_norm": 0.4220770299434662, "seconds": 0.3229226189999963, "nonpadding_tokens_per_second": 235378.37094031766, "ranks": [{"compute_seconds": 0.3198303170000827, "nonpadding_tokens": 14648, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3205856079999876, "nonpadding_tokens": 16317, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3173977270000705, "nonpadding_tokens": 9516, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31810452800004896, "nonpadding_tokens": 5253, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31683066699997653, "nonpadding_tokens": 9766, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3198319159999983, "nonpadding_tokens": 9134, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31651194999994914, "nonpadding_tokens": 5700, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3167115180000337, "nonpadding_tokens": 5675, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003609960000403589, "prefetch": {"data_wait_seconds": 0.01993675800140693, "consumed_batches": 2121, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2671, "tokens": 195472990, "loss_tokens": 112637, "loss": 1.9468347434679545, "lr": 0.00029917373021893004, "optimizer_lrs": [0.019944915347928673, 0.00029917373021893004], "gradient_norm": 0.4987291693687439, "seconds": 0.3657148549999647, "nonpadding_tokens_per_second": 307991.31744323287, "ranks": [{"compute_seconds": 0.3593136189999768, "nonpadding_tokens": 22313, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3591874800000596, "nonpadding_tokens": 8308, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3616909019999639, "nonpadding_tokens": 17064, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3588510849999693, "nonpadding_tokens": 10268, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3628216969999585, "nonpadding_tokens": 5336, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35992209999994884, "nonpadding_tokens": 19494, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35691516900010356, "nonpadding_tokens": 17354, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3633687510000527, "nonpadding_tokens": 12500, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003940629999306111, "prefetch": {"data_wait_seconds": 0.01994497800133388, "consumed_batches": 2122, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2672, "tokens": 195598450, "loss_tokens": 125460, "loss": 1.7905880111190817, "lr": 0.00029917254946968053, "optimizer_lrs": [0.019944836631312038, 0.00029917254946968053], "gradient_norm": 0.4356633424758911, "seconds": 0.3794744559999117, "nonpadding_tokens_per_second": 330615.138954252, "ranks": [{"compute_seconds": 0.3750787599999512, "nonpadding_tokens": 20781, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.376652052000054, "nonpadding_tokens": 26965, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3735822439999765, "nonpadding_tokens": 19444, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.37656988699995964, "nonpadding_tokens": 26372, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.37693495900009566, "nonpadding_tokens": 11890, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3754277440000351, "nonpadding_tokens": 3192, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3728643860000602, "nonpadding_tokens": 10236, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.37707046700006686, "nonpadding_tokens": 6580, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039956999989954056, "prefetch": {"data_wait_seconds": 0.019953472001319028, "consumed_batches": 2123, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2673, "tokens": 195637203, "loss_tokens": 38753, "loss": 2.35300049351018, "lr": 0.000299172184581341, "optimizer_lrs": [0.019944812305422735, 0.000299172184581341], "gradient_norm": 0.4744260311126709, "seconds": 0.2807276799999272, "nonpadding_tokens_per_second": 138044.81268113657, "ranks": [{"compute_seconds": 0.2784983069999498, "nonpadding_tokens": 2476, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27751962899992577, "nonpadding_tokens": 3515, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27745890999995027, "nonpadding_tokens": 3616, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2753075670000271, "nonpadding_tokens": 4091, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2739904809999416, "nonpadding_tokens": 2493, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27531859299995176, "nonpadding_tokens": 13127, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.273570540000037, "nonpadding_tokens": 3825, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2742476870000701, "nonpadding_tokens": 5610, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037844500002393033, "prefetch": {"data_wait_seconds": 0.019962342001306865, "consumed_batches": 2124, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2674, "tokens": 195688473, "loss_tokens": 51270, "loss": 2.287250249902477, "lr": 0.0002991717017129073, "optimizer_lrs": [0.01994478011419382, 0.0002991717017129073], "gradient_norm": 0.39155858755111694, "seconds": 0.29216801799998393, "nonpadding_tokens_per_second": 175481.21916616766, "ranks": [{"compute_seconds": 0.28739359600001535, "nonpadding_tokens": 17012, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28980816299997514, "nonpadding_tokens": 4263, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28864246400007687, "nonpadding_tokens": 4338, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.289273038000033, "nonpadding_tokens": 6429, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2855485119999912, "nonpadding_tokens": 7238, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2870383409999704, "nonpadding_tokens": 5418, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28985794400000486, "nonpadding_tokens": 4862, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28620594500000607, "nonpadding_tokens": 1710, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037082300002566626, "prefetch": {"data_wait_seconds": 0.019971390001344957, "consumed_batches": 2125, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2675, "tokens": 195766013, "loss_tokens": 77540, "loss": 2.252572059582151, "lr": 0.0002991709711630795, "optimizer_lrs": [0.01994473141087197, 0.0002991709711630795], "gradient_norm": 0.4261796772480011, "seconds": 0.37063499499993213, "nonpadding_tokens_per_second": 209208.52333443094, "ranks": [{"compute_seconds": 0.36473879399989073, "nonpadding_tokens": 10449, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36644468500003313, "nonpadding_tokens": 3612, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36341435700001057, "nonpadding_tokens": 5635, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.36797036199993727, "nonpadding_tokens": 3724, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36799645399992187, "nonpadding_tokens": 14682, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36521416300001874, "nonpadding_tokens": 20532, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36253803399995377, "nonpadding_tokens": 6266, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.36832608299994263, "nonpadding_tokens": 12640, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000342728000077841, "prefetch": {"data_wait_seconds": 0.01997923500137233, "consumed_batches": 2126, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2676, "tokens": 195841994, "loss_tokens": 75981, "loss": 2.4897192636975034, "lr": 0.00029917025499008815, "optimizer_lrs": [0.019944683666005877, 0.00029917025499008815], "gradient_norm": 0.43642717599868774, "seconds": 0.26384757400001035, "nonpadding_tokens_per_second": 287973.08555126993, "ranks": [{"compute_seconds": 0.2614981850000504, "nonpadding_tokens": 5176, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2577352370000199, "nonpadding_tokens": 10262, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25822073000006185, "nonpadding_tokens": 2764, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2587876030000871, "nonpadding_tokens": 18265, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25733775399999104, "nonpadding_tokens": 11081, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2599972359998901, "nonpadding_tokens": 11999, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25751718599997275, "nonpadding_tokens": 12576, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25775721200000135, "nonpadding_tokens": 3858, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003468959999963772, "prefetch": {"data_wait_seconds": 0.01998724600134949, "consumed_batches": 2127, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2677, "tokens": 195915858, "loss_tokens": 73864, "loss": 2.5820168654554316, "lr": 0.0002991695584757828, "optimizer_lrs": [0.019944637231718857, 0.0002991695584757828], "gradient_norm": 0.46427521109580994, "seconds": 0.37651470000002973, "nonpadding_tokens_per_second": 196178.26342502475, "ranks": [{"compute_seconds": 0.3731431269999348, "nonpadding_tokens": 23644, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36993677899999966, "nonpadding_tokens": 11561, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36996534299998984, "nonpadding_tokens": 4544, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3705682389999083, "nonpadding_tokens": 10350, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.37376281100000597, "nonpadding_tokens": 2533, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3716176450000148, "nonpadding_tokens": 4164, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3736384820000467, "nonpadding_tokens": 10531, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3741255730000148, "nonpadding_tokens": 6537, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004437039999629633, "prefetch": {"data_wait_seconds": 0.01999710400134802, "consumed_batches": 2128, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2678, "tokens": 195970225, "loss_tokens": 54367, "loss": 2.0663975561921752, "lr": 0.0002991690456259395, "optimizer_lrs": [0.019944603041729302, 0.0002991690456259395], "gradient_norm": 0.4787375032901764, "seconds": 0.2712704380001014, "nonpadding_tokens_per_second": 200416.23555007373, "ranks": [{"compute_seconds": 0.2658179159999463, "nonpadding_tokens": 4655, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26703883999994105, "nonpadding_tokens": 10512, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2676540599999271, "nonpadding_tokens": 9977, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2684520139999904, "nonpadding_tokens": 2740, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26322030000005725, "nonpadding_tokens": 6944, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26564220000000205, "nonpadding_tokens": 12261, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26817364099997576, "nonpadding_tokens": 4506, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2688838089999308, "nonpadding_tokens": 2772, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038341499998750805, "prefetch": {"data_wait_seconds": 0.02000729700144177, "consumed_batches": 2129, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2679, "tokens": 196048581, "loss_tokens": 78356, "loss": 2.173480054175813, "lr": 0.0002991683062076019, "optimizer_lrs": [0.019944553747173462, 0.0002991683062076019], "gradient_norm": 0.5849664807319641, "seconds": 0.2973243359999742, "nonpadding_tokens_per_second": 263537.12263905234, "ranks": [{"compute_seconds": 0.292215183000053, "nonpadding_tokens": 8270, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29452662500000315, "nonpadding_tokens": 7809, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29155100800005584, "nonpadding_tokens": 8953, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29471940400003405, "nonpadding_tokens": 22751, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2903182149999566, "nonpadding_tokens": 5522, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29180053599998246, "nonpadding_tokens": 3286, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29423675900000035, "nonpadding_tokens": 10213, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29491234099998564, "nonpadding_tokens": 11552, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033747800000583084, "prefetch": {"data_wait_seconds": 0.020016118001493055, "consumed_batches": 2130, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2680, "tokens": 196119889, "loss_tokens": 71308, "loss": 2.353430496928816, "lr": 0.0002991676330138747, "optimizer_lrs": [0.019944508867591648, 0.0002991676330138747], "gradient_norm": 0.4507497251033783, "seconds": 0.27170059200000196, "nonpadding_tokens_per_second": 262450.6611306886, "ranks": [{"compute_seconds": 0.2671523449999995, "nonpadding_tokens": 1602, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.269213786000023, "nonpadding_tokens": 6124, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2684260120000772, "nonpadding_tokens": 13539, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26884852500006673, "nonpadding_tokens": 6876, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26517597100007606, "nonpadding_tokens": 2183, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2681964449999441, "nonpadding_tokens": 17066, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2693761320000476, "nonpadding_tokens": 10307, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2659779470000103, "nonpadding_tokens": 13611, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038298399999803223, "prefetch": {"data_wait_seconds": 0.020025532001525903, "consumed_batches": 2131, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2681, "tokens": 196193107, "loss_tokens": 73218, "loss": 2.3470590394438524, "lr": 0.00029916694150600707, "optimizer_lrs": [0.01994446276706714, 0.00029916694150600707], "gradient_norm": 0.45332327485084534, "seconds": 0.3118941290000521, "nonpadding_tokens_per_second": 234752.735598905, "ranks": [{"compute_seconds": 0.3083305139999766, "nonpadding_tokens": 12681, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30695641299996623, "nonpadding_tokens": 9544, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3036491010000191, "nonpadding_tokens": 14504, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30426184899999953, "nonpadding_tokens": 9913, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3026717269999608, "nonpadding_tokens": 1976, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30530503600004977, "nonpadding_tokens": 9963, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30845744699990973, "nonpadding_tokens": 3584, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3094625079999105, "nonpadding_tokens": 11053, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003656330000012531, "prefetch": {"data_wait_seconds": 0.02003423200149257, "consumed_batches": 2132, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2682, "tokens": 196255527, "loss_tokens": 62420, "loss": 2.612598876561999, "lr": 0.00029916635175389155, "optimizer_lrs": [0.019944423450259437, 0.00029916635175389155], "gradient_norm": 0.45386385917663574, "seconds": 0.34469356800002515, "nonpadding_tokens_per_second": 181088.38050611797, "ranks": [{"compute_seconds": 0.34172909999995227, "nonpadding_tokens": 18953, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3411397350000698, "nonpadding_tokens": 1821, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34018204599999535, "nonpadding_tokens": 12848, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3379551049999918, "nonpadding_tokens": 4656, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3365626979999661, "nonpadding_tokens": 5713, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33807336899997154, "nonpadding_tokens": 3859, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33650890000001255, "nonpadding_tokens": 2874, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3377533959999255, "nonpadding_tokens": 11696, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033677800001896685, "prefetch": {"data_wait_seconds": 0.02004275900146979, "consumed_batches": 2133, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2683, "tokens": 196342689, "loss_tokens": 87162, "loss": 2.448890607432138, "lr": 0.0002991655278881411, "optimizer_lrs": [0.019944368525876072, 0.0002991655278881411], "gradient_norm": 0.3855598568916321, "seconds": 0.3419429330000412, "nonpadding_tokens_per_second": 254902.18275687977, "ranks": [{"compute_seconds": 0.2716082079999751, "nonpadding_tokens": 16415, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29686526099999355, "nonpadding_tokens": 32168, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29677829700005987, "nonpadding_tokens": 2505, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2972536150000451, "nonpadding_tokens": 5393, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29590316700000585, "nonpadding_tokens": 3110, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29832190199999786, "nonpadding_tokens": 7843, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2956437000000278, "nonpadding_tokens": 2501, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30062630400004764, "nonpadding_tokens": 17227, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01087518999997883, "prefetch": {"data_wait_seconds": 0.0200505800015236, "consumed_batches": 2134, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2684, "tokens": 196399513, "loss_tokens": 56824, "loss": 2.426175284650852, "lr": 0.00029916499056240444, "optimizer_lrs": [0.019944332704160298, 0.00029916499056240444], "gradient_norm": 0.5802751779556274, "seconds": 0.3071940209999866, "nonpadding_tokens_per_second": 184977.55853133116, "ranks": [{"compute_seconds": 0.30320280099999763, "nonpadding_tokens": 6744, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30472452000003614, "nonpadding_tokens": 14101, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3014056879999316, "nonpadding_tokens": 6284, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3045368519999556, "nonpadding_tokens": 5792, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30487206300006164, "nonpadding_tokens": 8691, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3021573550000767, "nonpadding_tokens": 8138, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30478206899999805, "nonpadding_tokens": 4196, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30097295300004134, "nonpadding_tokens": 2878, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039489900007083634, "prefetch": {"data_wait_seconds": 0.020059254001537852, "consumed_batches": 2135, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2685, "tokens": 196449658, "loss_tokens": 50145, "loss": 2.436296459018845, "lr": 0.0002991645162498583, "optimizer_lrs": [0.019944301083323887, 0.0002991645162498583], "gradient_norm": 0.4572627544403076, "seconds": 0.3512068150000687, "nonpadding_tokens_per_second": 142779.12004637552, "ranks": [{"compute_seconds": 0.3490516529999468, "nonpadding_tokens": 8979, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34372157400002834, "nonpadding_tokens": 7988, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3468962659999306, "nonpadding_tokens": 10602, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3468851600000562, "nonpadding_tokens": 5222, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34513174300002447, "nonpadding_tokens": 5383, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3444501089999221, "nonpadding_tokens": 5209, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34672518599995783, "nonpadding_tokens": 4810, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3453366889999643, "nonpadding_tokens": 1952, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003335610000476663, "prefetch": {"data_wait_seconds": 0.020067491001512394, "consumed_batches": 2136, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2686, "tokens": 196510807, "loss_tokens": 61149, "loss": 2.282228399074392, "lr": 0.00029916393767077183, "optimizer_lrs": [0.01994426251138479, 0.00029916393767077183], "gradient_norm": 0.46869802474975586, "seconds": 0.2829003190000776, "nonpadding_tokens_per_second": 216150.33951228322, "ranks": [{"compute_seconds": 0.28021047100003216, "nonpadding_tokens": 7634, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2785103229999777, "nonpadding_tokens": 4035, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27565580400005274, "nonpadding_tokens": 5381, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.280188126999974, "nonpadding_tokens": 22946, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2748068879999437, "nonpadding_tokens": 3660, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27778081799999654, "nonpadding_tokens": 5725, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2805070520000754, "nonpadding_tokens": 2307, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2752439839999852, "nonpadding_tokens": 9461, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034550700002000667, "prefetch": {"data_wait_seconds": 0.02007597100153191, "consumed_batches": 2137, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2687, "tokens": 196579307, "loss_tokens": 68500, "loss": 2.606730155109489, "lr": 0.0002991632893009701, "optimizer_lrs": [0.01994421928673134, 0.0002991632893009701], "gradient_norm": 0.46642398834228516, "seconds": 0.329487004999919, "nonpadding_tokens_per_second": 207898.9427823317, "ranks": [{"compute_seconds": 0.3263954939999394, "nonpadding_tokens": 7125, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3271373659999881, "nonpadding_tokens": 16183, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32623902700004237, "nonpadding_tokens": 6712, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32660333399996944, "nonpadding_tokens": 14066, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3231748929999867, "nonpadding_tokens": 10099, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3264328330000126, "nonpadding_tokens": 6974, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.323334573000011, "nonpadding_tokens": 4066, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.323875562000012, "nonpadding_tokens": 3275, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041680900005758303, "prefetch": {"data_wait_seconds": 0.020096674001436554, "consumed_batches": 2138, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2688, "tokens": 196642470, "loss_tokens": 63163, "loss": 2.2480259408197836, "lr": 0.0002991626912252119, "optimizer_lrs": [0.019944179415014127, 0.0002991626912252119], "gradient_norm": 0.46147170662879944, "seconds": 0.35797275699997044, "nonpadding_tokens_per_second": 176446.3880697078, "ranks": [{"compute_seconds": 0.35133928000004744, "nonpadding_tokens": 11166, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35326605599993854, "nonpadding_tokens": 11851, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35392027699992923, "nonpadding_tokens": 8451, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35083980999991127, "nonpadding_tokens": 5000, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3555078459999095, "nonpadding_tokens": 3936, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35208952400000726, "nonpadding_tokens": 10163, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35458818099994005, "nonpadding_tokens": 2751, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3555542010000181, "nonpadding_tokens": 9845, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003439690000277551, "prefetch": {"data_wait_seconds": 0.020105685001524876, "consumed_batches": 2139, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2689, "tokens": 196742695, "loss_tokens": 100225, "loss": 2.4331326390621104, "lr": 0.0002991617417808365, "optimizer_lrs": [0.019944116118722436, 0.0002991617417808365], "gradient_norm": 0.5551549792289734, "seconds": 0.3219092099999443, "nonpadding_tokens_per_second": 311345.54988351325, "ranks": [{"compute_seconds": 0.31386932700002035, "nonpadding_tokens": 13896, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31578279200005, "nonpadding_tokens": 10925, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3181274980000808, "nonpadding_tokens": 21125, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31328644999996413, "nonpadding_tokens": 9802, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3117352200000596, "nonpadding_tokens": 9249, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3165301219999037, "nonpadding_tokens": 7584, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3115318350000962, "nonpadding_tokens": 14708, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31959116600000925, "nonpadding_tokens": 12936, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038836999999602995, "prefetch": {"data_wait_seconds": 0.020113901001494128, "consumed_batches": 2140, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2690, "tokens": 196801856, "loss_tokens": 59161, "loss": 2.3294854295904397, "lr": 0.0002991611810893144, "optimizer_lrs": [0.019944078739287628, 0.0002991611810893144], "gradient_norm": 0.42420995235443115, "seconds": 0.26436323600000833, "nonpadding_tokens_per_second": 223786.79008150034, "ranks": [{"compute_seconds": 0.258733738999922, "nonpadding_tokens": 10188, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2595678760000055, "nonpadding_tokens": 9532, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25652103499999157, "nonpadding_tokens": 10806, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26017670400005954, "nonpadding_tokens": 997, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26193646400008674, "nonpadding_tokens": 2580, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25869243099998585, "nonpadding_tokens": 3941, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2553050940000503, "nonpadding_tokens": 10655, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25637363599992113, "nonpadding_tokens": 10462, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003382799999371855, "prefetch": {"data_wait_seconds": 0.020121900001484505, "consumed_batches": 2141, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2691, "tokens": 196885191, "loss_tokens": 83335, "loss": 2.397364927701446, "lr": 0.00029916039097450226, "optimizer_lrs": [0.01994402606496682, 0.00029916039097450226], "gradient_norm": 0.37994223833084106, "seconds": 0.4730785740000556, "nonpadding_tokens_per_second": 176154.6698159917, "ranks": [{"compute_seconds": 0.45000744699996176, "nonpadding_tokens": 22318, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4651302710000209, "nonpadding_tokens": 3448, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.46180468699992616, "nonpadding_tokens": 5075, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.462610161999919, "nonpadding_tokens": 9817, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4670929530000194, "nonpadding_tokens": 8695, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4635857959999612, "nonpadding_tokens": 10356, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4607057399999803, "nonpadding_tokens": 9470, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4612572739999905, "nonpadding_tokens": 14156, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004372590000230048, "prefetch": {"data_wait_seconds": 0.020133695001462115, "consumed_batches": 2142, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2692, "tokens": 196986814, "loss_tokens": 101623, "loss": 2.538351615775956, "lr": 0.0002991594269659568, "optimizer_lrs": [0.019943961797730454, 0.0002991594269659568], "gradient_norm": 1.1486481428146362, "seconds": 0.29356173700000454, "nonpadding_tokens_per_second": 346172.49863185827, "ranks": [{"compute_seconds": 0.2909166140000252, "nonpadding_tokens": 4862, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28926418300000023, "nonpadding_tokens": 12699, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28999573999999484, "nonpadding_tokens": 2054, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2904345930001, "nonpadding_tokens": 13734, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2856738169999744, "nonpadding_tokens": 9604, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28835081700003684, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2910707830000092, "nonpadding_tokens": 21474, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.286357300000077, "nonpadding_tokens": 4428, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047021699992910726, "prefetch": {"data_wait_seconds": 0.020142821001471134, "consumed_batches": 2143, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2693, "tokens": 197085962, "loss_tokens": 99148, "loss": 2.56751759364788, "lr": 0.00029915848590417757, "optimizer_lrs": [0.019943899060278505, 0.00029915848590417757], "gradient_norm": 0.7906289100646973, "seconds": 0.3110708210000439, "nonpadding_tokens_per_second": 318731.276952479, "ranks": [{"compute_seconds": 0.2978305880000107, "nonpadding_tokens": 5542, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3072177220000185, "nonpadding_tokens": 7414, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30374865499993575, "nonpadding_tokens": 7605, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30431596699997954, "nonpadding_tokens": 17571, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30289514300000064, "nonpadding_tokens": 11479, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30520109400004003, "nonpadding_tokens": 16091, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30774072799999885, "nonpadding_tokens": 21622, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3085652070000151, "nonpadding_tokens": 11824, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000345633999927486, "prefetch": {"data_wait_seconds": 0.020151211001532943, "consumed_batches": 2144, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2694, "tokens": 197140346, "loss_tokens": 54384, "loss": 2.2064926897157253, "lr": 0.00029915796949634923, "optimizer_lrs": [0.01994386463308995, 0.00029915796949634923], "gradient_norm": 0.4148775339126587, "seconds": 0.2715742409999393, "nonpadding_tokens_per_second": 200254.63313367835, "ranks": [{"compute_seconds": 0.2691185850000011, "nonpadding_tokens": 10869, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26460780700006126, "nonpadding_tokens": 2665, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26176261499995235, "nonpadding_tokens": 1853, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2622156840000116, "nonpadding_tokens": 11778, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2605840509999098, "nonpadding_tokens": 7288, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26338895900005355, "nonpadding_tokens": 4559, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2610065339999892, "nonpadding_tokens": 13636, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26853888699997697, "nonpadding_tokens": 1736, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040067999998427695, "prefetch": {"data_wait_seconds": 0.02016119100149183, "consumed_batches": 2145, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2695, "tokens": 197242689, "loss_tokens": 102343, "loss": 2.656308931729576, "lr": 0.0002991569972616357, "optimizer_lrs": [0.019943799817442383, 0.0002991569972616357], "gradient_norm": 0.6859567761421204, "seconds": 0.3121408289999863, "nonpadding_tokens_per_second": 327874.441571386, "ranks": [{"compute_seconds": 0.30768200700003945, "nonpadding_tokens": 4158, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30865418400003364, "nonpadding_tokens": 13469, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30726919699998234, "nonpadding_tokens": 10778, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3061154129999295, "nonpadding_tokens": 23097, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3097865390000152, "nonpadding_tokens": 10905, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30789590699998826, "nonpadding_tokens": 3309, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30429495800001405, "nonpadding_tokens": 23621, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30969133799999327, "nonpadding_tokens": 13006, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038567599995076307, "prefetch": {"data_wait_seconds": 0.020170633001569627, "consumed_batches": 2146, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2696, "tokens": 197312541, "loss_tokens": 69852, "loss": 2.4548250229055717, "lr": 0.00029915633336278266, "optimizer_lrs": [0.019943755557518844, 0.00029915633336278266], "gradient_norm": 0.7056407928466797, "seconds": 0.3618875309999794, "nonpadding_tokens_per_second": 193021.2953372079, "ranks": [{"compute_seconds": 0.32708673599995564, "nonpadding_tokens": 10190, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32740346000002774, "nonpadding_tokens": 16319, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33007640000005267, "nonpadding_tokens": 4416, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32732677100000274, "nonpadding_tokens": 5389, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3255279919999339, "nonpadding_tokens": 2166, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3282526120000284, "nonpadding_tokens": 16825, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32556235200001993, "nonpadding_tokens": 10618, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.326475088000052, "nonpadding_tokens": 3929, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.002040415999999823, "prefetch": {"data_wait_seconds": 0.02017970900153898, "consumed_batches": 2147, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2697, "tokens": 197377607, "loss_tokens": 65066, "loss": 2.3900496226908063, "lr": 0.00029915571471747705, "optimizer_lrs": [0.01994371431449847, 0.00029915571471747705], "gradient_norm": 0.43393421173095703, "seconds": 0.255231429000105, "nonpadding_tokens_per_second": 254929.41937010916, "ranks": [{"compute_seconds": 0.2528009010000005, "nonpadding_tokens": 2795, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25018697099994824, "nonpadding_tokens": 12407, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2473242940000091, "nonpadding_tokens": 18272, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25232098599997244, "nonpadding_tokens": 1703, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24654546299996127, "nonpadding_tokens": 3953, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24973014399995463, "nonpadding_tokens": 11942, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2524322799999936, "nonpadding_tokens": 5097, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24685868099993513, "nonpadding_tokens": 8897, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034809800001767144, "prefetch": {"data_wait_seconds": 0.020189254001550694, "consumed_batches": 2148, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2698, "tokens": 197426361, "loss_tokens": 48754, "loss": 2.2300151013250193, "lr": 0.0002991552510179782, "optimizer_lrs": [0.01994368340119855, 0.0002991552510179782], "gradient_norm": 0.396188348531723, "seconds": 0.24400202300000728, "nonpadding_tokens_per_second": 199809.8187898981, "ranks": [{"compute_seconds": 0.24125684000000547, "nonpadding_tokens": 4265, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2399499130000322, "nonpadding_tokens": 15549, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23648264400003427, "nonpadding_tokens": 2139, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23708727799998996, "nonpadding_tokens": 4258, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24094964400001118, "nonpadding_tokens": 3990, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23773656100001972, "nonpadding_tokens": 10827, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23574055900007806, "nonpadding_tokens": 3939, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23654958200006604, "nonpadding_tokens": 3787, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004493330000059359, "prefetch": {"data_wait_seconds": 0.020211200001497787, "consumed_batches": 2149, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2699, "tokens": 197506515, "loss_tokens": 80154, "loss": 2.327620307470619, "lr": 0.0002991544883971288, "optimizer_lrs": [0.01994363255980859, 0.0002991544883971288], "gradient_norm": 0.3818681836128235, "seconds": 0.2595781259999512, "nonpadding_tokens_per_second": 308785.64860282204, "ranks": [{"compute_seconds": 0.25599868100005096, "nonpadding_tokens": 3781, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25679213300008996, "nonpadding_tokens": 11106, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25349642000003314, "nonpadding_tokens": 4290, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2562423829999716, "nonpadding_tokens": 4868, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25272824399996807, "nonpadding_tokens": 18420, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.255164063000052, "nonpadding_tokens": 5168, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2567041159999235, "nonpadding_tokens": 23779, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2572132389999524, "nonpadding_tokens": 8742, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004043179999371205, "prefetch": {"data_wait_seconds": 0.020219050001472283, "consumed_batches": 2150, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2700, "tokens": 197570496, "loss_tokens": 63981, "loss": 2.265779098482362, "lr": 0.0002991538794072325, "optimizer_lrs": [0.01994359196048217, 0.0002991538794072325], "gradient_norm": 0.38866928219795227, "seconds": 0.2597864680000157, "nonpadding_tokens_per_second": 246283.035804606, "ranks": [{"compute_seconds": 0.25712229300006584, "nonpadding_tokens": 3765, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25740111799996157, "nonpadding_tokens": 18208, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25451423699996667, "nonpadding_tokens": 4049, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25685761600004753, "nonpadding_tokens": 3288, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25313790500001687, "nonpadding_tokens": 2738, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2560883300000114, "nonpadding_tokens": 10887, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25308552699993925, "nonpadding_tokens": 6711, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25347268500001974, "nonpadding_tokens": 14335, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004610720000073343, "prefetch": {"data_wait_seconds": 0.020228515001420533, "consumed_batches": 2151, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2701, "tokens": 197629505, "loss_tokens": 59009, "loss": 2.1725209554898406, "lr": 0.0002991533175485558, "optimizer_lrs": [0.019943554503237055, 0.0002991533175485558], "gradient_norm": 0.45009398460388184, "seconds": 0.3575050290000945, "nonpadding_tokens_per_second": 165057.81796984008, "ranks": [{"compute_seconds": 0.35262923300001603, "nonpadding_tokens": 18844, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35462922600004276, "nonpadding_tokens": 11419, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3520880630001102, "nonpadding_tokens": 10189, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3523136240000895, "nonpadding_tokens": 4017, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35508220999997775, "nonpadding_tokens": 2164, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3524540399999978, "nonpadding_tokens": 2472, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35077147499998773, "nonpadding_tokens": 1257, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35519168200005424, "nonpadding_tokens": 8647, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037736100000529404, "prefetch": {"data_wait_seconds": 0.0202376730014322, "consumed_batches": 2152, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2702, "tokens": 197675892, "loss_tokens": 46387, "loss": 2.367322404445211, "lr": 0.0002991528757407043, "optimizer_lrs": [0.019943525049380288, 0.0002991528757407043], "gradient_norm": 0.4849282205104828, "seconds": 0.2490800030000173, "nonpadding_tokens_per_second": 186233.3364432984, "ranks": [{"compute_seconds": 0.24687289500002407, "nonpadding_tokens": 6252, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24314281199997367, "nonpadding_tokens": 8235, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24337829000000966, "nonpadding_tokens": 5060, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24356961800003774, "nonpadding_tokens": 5521, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2463059990000147, "nonpadding_tokens": 3116, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24526081699991664, "nonpadding_tokens": 11834, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24255989400001, "nonpadding_tokens": 2058, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2432950229999733, "nonpadding_tokens": 4311, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003875419999985752, "prefetch": {"data_wait_seconds": 0.02024733600144657, "consumed_batches": 2153, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2703, "tokens": 197725782, "loss_tokens": 49890, "loss": 2.305559888003608, "lr": 0.0002991524004406992, "optimizer_lrs": [0.019943493362713283, 0.0002991524004406992], "gradient_norm": 0.4093659520149231, "seconds": 0.2921505959999422, "nonpadding_tokens_per_second": 170768.0924943582, "ranks": [{"compute_seconds": 0.2878689389999636, "nonpadding_tokens": 8823, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28985153500002525, "nonpadding_tokens": 5253, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28712367399998584, "nonpadding_tokens": 1958, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28756048900004316, "nonpadding_tokens": 3990, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2898516280000649, "nonpadding_tokens": 6763, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2886824129999468, "nonpadding_tokens": 15845, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28573167199999716, "nonpadding_tokens": 2486, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28609616400001414, "nonpadding_tokens": 4772, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003404550000141171, "prefetch": {"data_wait_seconds": 0.020255058001410475, "consumed_batches": 2154, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2704, "tokens": 197775461, "loss_tokens": 49679, "loss": 2.4840839124680447, "lr": 0.0002991519270188424, "optimizer_lrs": [0.01994346180125616, 0.0002991519270188424], "gradient_norm": 0.4184935390949249, "seconds": 0.2621512890000304, "nonpadding_tokens_per_second": 189505.07620809085, "ranks": [{"compute_seconds": 0.2464693009999337, "nonpadding_tokens": 6781, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25780261199997767, "nonpadding_tokens": 3816, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25445938699999715, "nonpadding_tokens": 7898, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25989317900007336, "nonpadding_tokens": 2615, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.25980917900005807, "nonpadding_tokens": 7806, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2562053440000227, "nonpadding_tokens": 5280, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25382900199997493, "nonpadding_tokens": 6435, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2542616399999815, "nonpadding_tokens": 9048, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003367530000559782, "prefetch": {"data_wait_seconds": 0.020262551001408724, "consumed_batches": 2155, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2705, "tokens": 197844993, "loss_tokens": 69532, "loss": 1.9850188132802162, "lr": 0.0002991512641842306, "optimizer_lrs": [0.01994341761228204, 0.0002991512641842306], "gradient_norm": 0.48947596549987793, "seconds": 0.33112451699992107, "nonpadding_tokens_per_second": 209987.47126905306, "ranks": [{"compute_seconds": 0.3264071359999434, "nonpadding_tokens": 9341, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3278161599999976, "nonpadding_tokens": 7620, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3268089709999913, "nonpadding_tokens": 17639, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3251733399999921, "nonpadding_tokens": 8357, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3238564860000679, "nonpadding_tokens": 2225, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32516778100000465, "nonpadding_tokens": 17100, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3276693669999986, "nonpadding_tokens": 3123, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32875988800003597, "nonpadding_tokens": 4127, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036706200000935496, "prefetch": {"data_wait_seconds": 0.02027082100137534, "consumed_batches": 2156, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2706, "tokens": 197914929, "loss_tokens": 69936, "loss": 2.4377403983642187, "lr": 0.00029915059723800366, "optimizer_lrs": [0.019943373149200246, 0.00029915059723800366], "gradient_norm": 0.455227792263031, "seconds": 0.31848676499998874, "nonpadding_tokens_per_second": 219588.40267664645, "ranks": [{"compute_seconds": 0.31604287100003603, "nonpadding_tokens": 9274, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3126474059999964, "nonpadding_tokens": 12644, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3116532650000181, "nonpadding_tokens": 5708, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3156624110000621, "nonpadding_tokens": 4910, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31058000300004096, "nonpadding_tokens": 6670, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3134198520000382, "nonpadding_tokens": 9651, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31040936299996247, "nonpadding_tokens": 9051, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31086969400007547, "nonpadding_tokens": 12028, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000405750000027183, "prefetch": {"data_wait_seconds": 0.020280773001289276, "consumed_batches": 2157, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2707, "tokens": 197978021, "loss_tokens": 63092, "loss": 2.062894141689913, "lr": 0.000299149995335698, "optimizer_lrs": [0.01994333302237987, 0.000299149995335698], "gradient_norm": 0.4659598171710968, "seconds": 0.3019062280000071, "nonpadding_tokens_per_second": 208978.7958928708, "ranks": [{"compute_seconds": 0.2973298470000145, "nonpadding_tokens": 4169, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2959598579999465, "nonpadding_tokens": 11102, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29923503199995594, "nonpadding_tokens": 2899, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2977428620000637, "nonpadding_tokens": 5746, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2958523999999443, "nonpadding_tokens": 3619, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.298372253000025, "nonpadding_tokens": 13737, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.299552134999999, "nonpadding_tokens": 15924, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29675949299996773, "nonpadding_tokens": 5896, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003886650000595182, "prefetch": {"data_wait_seconds": 0.02028983000123219, "consumed_batches": 2158, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2708, "tokens": 198044893, "loss_tokens": 66872, "loss": 2.6550235431122142, "lr": 0.00029914935713992535, "optimizer_lrs": [0.019943290475995024, 0.00029914935713992535], "gradient_norm": 0.4402662515640259, "seconds": 0.5314299440000241, "nonpadding_tokens_per_second": 125834.08359841495, "ranks": [{"compute_seconds": 0.5285319900000331, "nonpadding_tokens": 3368, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5271887760000027, "nonpadding_tokens": 12171, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.527870091000068, "nonpadding_tokens": 14229, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.5281844459999547, "nonpadding_tokens": 6310, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.5231455910000022, "nonpadding_tokens": 15863, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.525492168000028, "nonpadding_tokens": 8396, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.5231103039999425, "nonpadding_tokens": 5460, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.529016396999964, "nonpadding_tokens": 1075, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036783699999887176, "prefetch": {"data_wait_seconds": 0.02029930700128091, "consumed_batches": 2159, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2709, "tokens": 198091998, "loss_tokens": 47105, "loss": 2.2450035160810953, "lr": 0.0002991489074480443, "optimizer_lrs": [0.019943260496536285, 0.0002991489074480443], "gradient_norm": 0.4357525408267975, "seconds": 0.2703004139999621, "nonpadding_tokens_per_second": 174269.0634576927, "ranks": [{"compute_seconds": 0.26770308499999373, "nonpadding_tokens": 4598, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2680466639999395, "nonpadding_tokens": 3106, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2653117199999997, "nonpadding_tokens": 6952, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2668559420000065, "nonpadding_tokens": 2497, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26444653600003676, "nonpadding_tokens": 1477, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2671161600000005, "nonpadding_tokens": 5672, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26698948800003564, "nonpadding_tokens": 19290, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.265071077000016, "nonpadding_tokens": 3513, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037212000006547896, "prefetch": {"data_wait_seconds": 0.020309292001300605, "consumed_batches": 2160, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2710, "tokens": 198151255, "loss_tokens": 59257, "loss": 2.130194138456216, "lr": 0.0002991483415778257, "optimizer_lrs": [0.01994322277185505, 0.0002991483415778257], "gradient_norm": 1.2316054105758667, "seconds": 0.2944503950000126, "nonpadding_tokens_per_second": 201246.12160903186, "ranks": [{"compute_seconds": 0.2905416859999832, "nonpadding_tokens": 9738, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2915711680000186, "nonpadding_tokens": 6036, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28848961200003487, "nonpadding_tokens": 12823, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2896546119999357, "nonpadding_tokens": 7204, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2920572179999681, "nonpadding_tokens": 3717, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29076564700005747, "nonpadding_tokens": 3492, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2877254590000575, "nonpadding_tokens": 8654, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29210361599996304, "nonpadding_tokens": 7593, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003590810000559941, "prefetch": {"data_wait_seconds": 0.02031732300129363, "consumed_batches": 2161, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2711, "tokens": 198215529, "loss_tokens": 64274, "loss": 2.230921280377758, "lr": 0.0002991477275862095, "optimizer_lrs": [0.019943181839080632, 0.0002991477275862095], "gradient_norm": 0.542778730392456, "seconds": 0.2747975800000404, "nonpadding_tokens_per_second": 233895.80068350874, "ranks": [{"compute_seconds": 0.2707376780000459, "nonpadding_tokens": 6131, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27257263499996043, "nonpadding_tokens": 17048, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27188469800000803, "nonpadding_tokens": 4681, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2699253340000496, "nonpadding_tokens": 8539, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2684555799999089, "nonpadding_tokens": 5418, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27103468999996494, "nonpadding_tokens": 3970, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26828032299999904, "nonpadding_tokens": 8059, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.269244917999913, "nonpadding_tokens": 10428, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003932459999305138, "prefetch": {"data_wait_seconds": 0.020327387001273678, "consumed_batches": 2162, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2712, "tokens": 198332032, "loss_tokens": 116503, "loss": 2.2944908071036796, "lr": 0.00029914661410332806, "optimizer_lrs": [0.019943107606888538, 0.00029914661410332806], "gradient_norm": 0.5241506695747375, "seconds": 0.3067675910000389, "nonpadding_tokens_per_second": 379776.10222843004, "ranks": [{"compute_seconds": 0.30381608200002574, "nonpadding_tokens": 16341, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.301157885000066, "nonpadding_tokens": 4565, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30386594799995237, "nonpadding_tokens": 19894, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.302498521000075, "nonpadding_tokens": 20603, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3008564420000539, "nonpadding_tokens": 18111, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3035199129999455, "nonpadding_tokens": 6047, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30449655899997197, "nonpadding_tokens": 15276, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.301329485999986, "nonpadding_tokens": 15666, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037272899999152287, "prefetch": {"data_wait_seconds": 0.020337367001232565, "consumed_batches": 2163, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2713, "tokens": 198380976, "loss_tokens": 48944, "loss": 2.5661411191872343, "lr": 0.0002991461461026074, "optimizer_lrs": [0.019943076406840496, 0.0002991461461026074], "gradient_norm": 0.4382471442222595, "seconds": 0.22532261100002415, "nonpadding_tokens_per_second": 217217.43673560908, "ranks": [{"compute_seconds": 0.2225281260000429, "nonpadding_tokens": 4753, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2228480149999541, "nonpadding_tokens": 3942, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22163661699994464, "nonpadding_tokens": 7483, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2208415039999636, "nonpadding_tokens": 9299, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.223024814999917, "nonpadding_tokens": 3096, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22129347199995664, "nonpadding_tokens": 4957, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22278406899999936, "nonpadding_tokens": 4738, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21943186999999398, "nonpadding_tokens": 10676, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003468549999752213, "prefetch": {"data_wait_seconds": 0.02034692400123106, "consumed_batches": 2164, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2714, "tokens": 198451837, "loss_tokens": 70861, "loss": 2.082012231693033, "lr": 0.0002991454683057362, "optimizer_lrs": [0.019943031220382417, 0.0002991454683057362], "gradient_norm": 0.39658766984939575, "seconds": 0.3057615180000539, "nonpadding_tokens_per_second": 231752.51242698077, "ranks": [{"compute_seconds": 0.3023763899999494, "nonpadding_tokens": 10049, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3032115099999828, "nonpadding_tokens": 16726, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30219801199996255, "nonpadding_tokens": 6751, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3001650569999583, "nonpadding_tokens": 3075, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30345534600007795, "nonpadding_tokens": 7512, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30192515700002787, "nonpadding_tokens": 2682, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2991243779999877, "nonpadding_tokens": 17204, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2997288239999989, "nonpadding_tokens": 6862, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003632519999428041, "prefetch": {"data_wait_seconds": 0.020356445001269208, "consumed_batches": 2165, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2715, "tokens": 198524366, "loss_tokens": 72529, "loss": 2.176801693115857, "lr": 0.00029914477427657763, "optimizer_lrs": [0.019942984951771842, 0.00029914477427657763], "gradient_norm": 0.47838836908340454, "seconds": 0.33086338199996135, "nonpadding_tokens_per_second": 219211.32390531048, "ranks": [{"compute_seconds": 0.3284832550000374, "nonpadding_tokens": 12227, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3261987030000455, "nonpadding_tokens": 7399, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3237832759999719, "nonpadding_tokens": 3024, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3282174759999634, "nonpadding_tokens": 21849, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3227673369999593, "nonpadding_tokens": 3937, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32542517499996393, "nonpadding_tokens": 5167, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32260688999997456, "nonpadding_tokens": 12963, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32346048800002336, "nonpadding_tokens": 5963, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042170799997620634, "prefetch": {"data_wait_seconds": 0.020364221001344163, "consumed_batches": 2166, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2716, "tokens": 198574818, "loss_tokens": 50452, "loss": 2.046815072990169, "lr": 0.00029914429133636, "optimizer_lrs": [0.01994295275575733, 0.00029914429133636], "gradient_norm": 0.3972034156322479, "seconds": 0.2895752330000505, "nonpadding_tokens_per_second": 174227.60737273135, "ranks": [{"compute_seconds": 0.2837780059999204, "nonpadding_tokens": 2284, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2864525960000037, "nonpadding_tokens": 10572, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2856336629999987, "nonpadding_tokens": 15829, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2841309980000233, "nonpadding_tokens": 6385, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28662110199991275, "nonpadding_tokens": 2108, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2852405159999307, "nonpadding_tokens": 3548, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2821270079999749, "nonpadding_tokens": 5486, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2872629099999813, "nonpadding_tokens": 4240, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003902930000094784, "prefetch": {"data_wait_seconds": 0.020372754001371618, "consumed_batches": 2167, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2717, "tokens": 198631028, "loss_tokens": 56210, "loss": 2.1489384117594734, "lr": 0.00029914375311897385, "optimizer_lrs": [0.019942916874598256, 0.00029914375311897385], "gradient_norm": 0.42758992314338684, "seconds": 0.3156397149999748, "nonpadding_tokens_per_second": 178082.78657203985, "ranks": [{"compute_seconds": 0.25152587100001256, "nonpadding_tokens": 12108, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25397550499997124, "nonpadding_tokens": 15992, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25091162000001077, "nonpadding_tokens": 6293, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25346706299990274, "nonpadding_tokens": 6338, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2541223179999861, "nonpadding_tokens": 3473, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25144359900002655, "nonpadding_tokens": 1875, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2538269010000249, "nonpadding_tokens": 6781, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25455049699996835, "nonpadding_tokens": 3350, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0412465729999667, "prefetch": {"data_wait_seconds": 0.020381394001404374, "consumed_batches": 2168, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2718, "tokens": 198710203, "loss_tokens": 79175, "loss": 2.349639248500158, "lr": 0.00029914299472288703, "optimizer_lrs": [0.019942866314859135, 0.00029914299472288703], "gradient_norm": 0.4145074486732483, "seconds": 0.3320827630000167, "nonpadding_tokens_per_second": 238419.4809894304, "ranks": [{"compute_seconds": 0.32905612700005804, "nonpadding_tokens": 4316, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32606354799997916, "nonpadding_tokens": 13620, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3265647320000653, "nonpadding_tokens": 20077, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3290984399999388, "nonpadding_tokens": 10976, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3296881999999641, "nonpadding_tokens": 3146, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3283256370000345, "nonpadding_tokens": 3754, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3255739400000266, "nonpadding_tokens": 4539, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32597035399999186, "nonpadding_tokens": 18747, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000402385000029426, "prefetch": {"data_wait_seconds": 0.020389784001352496, "consumed_batches": 2169, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2719, "tokens": 198808258, "loss_tokens": 98055, "loss": 2.4754904772831576, "lr": 0.00029914205501648574, "optimizer_lrs": [0.019942803667765716, 0.00029914205501648574], "gradient_norm": 0.3757261633872986, "seconds": 0.3582026990000031, "nonpadding_tokens_per_second": 273741.65597786056, "ranks": [{"compute_seconds": 0.35508888900005786, "nonpadding_tokens": 10321, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3558256019999817, "nonpadding_tokens": 10607, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35289460800004235, "nonpadding_tokens": 6295, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3537572169999521, "nonpadding_tokens": 13947, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35202338900000996, "nonpadding_tokens": 17963, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3532636749999938, "nonpadding_tokens": 17021, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35165157599999475, "nonpadding_tokens": 14120, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3526944559999947, "nonpadding_tokens": 7781, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004029840000612239, "prefetch": {"data_wait_seconds": 0.02039923000138799, "consumed_batches": 2170, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2720, "tokens": 198893924, "loss_tokens": 85666, "loss": 2.1875300950785608, "lr": 0.0002991412336195085, "optimizer_lrs": [0.019942748907967236, 0.0002991412336195085], "gradient_norm": 0.4402141571044922, "seconds": 0.263996167000073, "nonpadding_tokens_per_second": 324497.13559657976, "ranks": [{"compute_seconds": 0.26087611100001595, "nonpadding_tokens": 2263, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26046030299994527, "nonpadding_tokens": 9798, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26023893300009604, "nonpadding_tokens": 15624, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25867362100007085, "nonpadding_tokens": 4361, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2611721559999296, "nonpadding_tokens": 5581, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25815251599999556, "nonpadding_tokens": 17827, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2570445830000381, "nonpadding_tokens": 18189, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2616839529999879, "nonpadding_tokens": 12023, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037117099998340564, "prefetch": {"data_wait_seconds": 0.02040776400140487, "consumed_batches": 2171, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2721, "tokens": 198983434, "loss_tokens": 89510, "loss": 2.0241191626633896, "lr": 0.0002991403749463286, "optimizer_lrs": [0.019942691663088572, 0.0002991403749463286], "gradient_norm": 0.40175458788871765, "seconds": 0.27457831700007773, "nonpadding_tokens_per_second": 325990.78098353505, "ranks": [{"compute_seconds": 0.2712745910000649, "nonpadding_tokens": 11840, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26913966200004324, "nonpadding_tokens": 9808, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2714976490000254, "nonpadding_tokens": 6580, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27020914100000937, "nonpadding_tokens": 14297, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2684814760000336, "nonpadding_tokens": 17099, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2697580689999768, "nonpadding_tokens": 5969, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27227334600001996, "nonpadding_tokens": 12512, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26944324399994457, "nonpadding_tokens": 11405, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036297400004059455, "prefetch": {"data_wait_seconds": 0.020417663001424557, "consumed_batches": 2172, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2722, "tokens": 199090923, "loss_tokens": 107489, "loss": 2.30360548521244, "lr": 0.0002991393432346533, "optimizer_lrs": [0.01994262288231022, 0.0002991393432346533], "gradient_norm": 0.4908338487148285, "seconds": 0.3439917150000156, "nonpadding_tokens_per_second": 312475.54901139153, "ranks": [{"compute_seconds": 0.341300674000081, "nonpadding_tokens": 11059, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3411991360000002, "nonpadding_tokens": 6380, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3403028870000071, "nonpadding_tokens": 10513, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33859279799992237, "nonpadding_tokens": 19960, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.337020680999899, "nonpadding_tokens": 16873, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3381466390000014, "nonpadding_tokens": 12156, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3410701710000694, "nonpadding_tokens": 19753, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3378417360000867, "nonpadding_tokens": 10795, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003808389999448991, "prefetch": {"data_wait_seconds": 0.020428019001386133, "consumed_batches": 2173, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2723, "tokens": 199168630, "loss_tokens": 77707, "loss": 2.1287749977479504, "lr": 0.00029913859699543714, "optimizer_lrs": [0.019942573133029145, 0.00029913859699543714], "gradient_norm": 0.4602908194065094, "seconds": 0.3115431209999997, "nonpadding_tokens_per_second": 249426.1460518657, "ranks": [{"compute_seconds": 0.30921440900010566, "nonpadding_tokens": 3498, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30532338900002287, "nonpadding_tokens": 7070, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30756405599993286, "nonpadding_tokens": 2612, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3062413599999445, "nonpadding_tokens": 5069, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30866742499995325, "nonpadding_tokens": 20038, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3068293909999511, "nonpadding_tokens": 9438, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3086343149999493, "nonpadding_tokens": 3069, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30512106100002256, "nonpadding_tokens": 26913, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003401769999982207, "prefetch": {"data_wait_seconds": 0.02043603400132099, "consumed_batches": 2174, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2724, "tokens": 199229468, "loss_tokens": 60838, "loss": 2.215850393668431, "lr": 0.0002991380125283566, "optimizer_lrs": [0.01994253416855711, 0.0002991380125283566], "gradient_norm": 0.38585659861564636, "seconds": 0.2933363689999169, "nonpadding_tokens_per_second": 207400.1263717055, "ranks": [{"compute_seconds": 0.2891684040000655, "nonpadding_tokens": 12778, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29092698900001324, "nonpadding_tokens": 8365, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28798222499995063, "nonpadding_tokens": 12210, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2904139250000526, "nonpadding_tokens": 3575, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2909073289999924, "nonpadding_tokens": 3701, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2881211769999936, "nonpadding_tokens": 3997, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.286654952000049, "nonpadding_tokens": 10656, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2910018930000433, "nonpadding_tokens": 5556, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003637749999825246, "prefetch": {"data_wait_seconds": 0.020444343001329912, "consumed_batches": 2175, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2725, "tokens": 199280626, "loss_tokens": 51158, "loss": 2.457268413542359, "lr": 0.0002991375209035404, "optimizer_lrs": [0.019942501393569362, 0.0002991375209035404], "gradient_norm": 0.38219568133354187, "seconds": 0.36970800699998563, "nonpadding_tokens_per_second": 138374.06556358945, "ranks": [{"compute_seconds": 0.3674148800000694, "nonpadding_tokens": 5893, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36455862900004377, "nonpadding_tokens": 9152, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36689237399991725, "nonpadding_tokens": 11065, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3643631270000469, "nonpadding_tokens": 2737, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36249411600010717, "nonpadding_tokens": 6083, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36516010999991977, "nonpadding_tokens": 3987, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36265436700000464, "nonpadding_tokens": 7509, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.36324554800000897, "nonpadding_tokens": 4732, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038649800001167023, "prefetch": {"data_wait_seconds": 0.0204532960012358, "consumed_batches": 2176, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2726, "tokens": 199354168, "loss_tokens": 73542, "loss": 2.1383053221288515, "lr": 0.0002991368139252708, "optimizer_lrs": [0.01994245426168472, 0.0002991368139252708], "gradient_norm": 0.3833966553211212, "seconds": 0.22836213699997643, "nonpadding_tokens_per_second": 322041.1271593924, "ranks": [{"compute_seconds": 0.2228919850000466, "nonpadding_tokens": 2576, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22170039599996016, "nonpadding_tokens": 2831, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22279056200000014, "nonpadding_tokens": 15943, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22524766700007604, "nonpadding_tokens": 9649, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22549980100006906, "nonpadding_tokens": 10923, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22238518699998622, "nonpadding_tokens": 13809, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22554690499998742, "nonpadding_tokens": 11815, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2259266219999745, "nonpadding_tokens": 5996, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003841010000087408, "prefetch": {"data_wait_seconds": 0.02046178100120244, "consumed_batches": 2177, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2727, "tokens": 199420857, "loss_tokens": 66689, "loss": 2.289205772316274, "lr": 0.0002991361725770556, "optimizer_lrs": [0.01994241150513704, 0.0002991361725770556], "gradient_norm": 0.4204101264476776, "seconds": 0.31443135600000005, "nonpadding_tokens_per_second": 212093.98721672018, "ranks": [{"compute_seconds": 0.31220350600005986, "nonpadding_tokens": 1945, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3118960110000444, "nonpadding_tokens": 5195, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30915401799995834, "nonpadding_tokens": 18726, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3109404240000231, "nonpadding_tokens": 9113, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3081912850000208, "nonpadding_tokens": 10400, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.310289465999972, "nonpadding_tokens": 4995, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30787941000005503, "nonpadding_tokens": 2117, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30885924599999726, "nonpadding_tokens": 14198, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000342316999990544, "prefetch": {"data_wait_seconds": 0.020469697001274767, "consumed_batches": 2178, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2728, "tokens": 199485189, "loss_tokens": 64332, "loss": 2.3204098951532672, "lr": 0.0002991355536711428, "optimizer_lrs": [0.019942370244742855, 0.0002991355536711428], "gradient_norm": 0.4067550301551819, "seconds": 0.2777422919999708, "nonpadding_tokens_per_second": 231624.78978897014, "ranks": [{"compute_seconds": 0.27490656600002694, "nonpadding_tokens": 5247, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27193823100003556, "nonpadding_tokens": 6081, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27484770800003844, "nonpadding_tokens": 6078, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27504501400005665, "nonpadding_tokens": 12467, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2717449919999808, "nonpadding_tokens": 8398, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27474174400003903, "nonpadding_tokens": 4896, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2754284820000521, "nonpadding_tokens": 7289, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2723906840000154, "nonpadding_tokens": 13876, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003953149999915695, "prefetch": {"data_wait_seconds": 0.02047913300130233, "consumed_batches": 2179, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2729, "tokens": 199601348, "loss_tokens": 116159, "loss": 2.0496753103504677, "lr": 0.0002991344356040707, "optimizer_lrs": [0.01994229570693805, 0.0002991344356040707], "gradient_norm": 0.4647659361362457, "seconds": 0.3728874349999387, "nonpadding_tokens_per_second": 311512.2396119866, "ranks": [{"compute_seconds": 0.3706344120000722, "nonpadding_tokens": 28664, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3669570679999197, "nonpadding_tokens": 25213, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36959022799999275, "nonpadding_tokens": 18780, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.370066508999912, "nonpadding_tokens": 19128, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.36663026100006846, "nonpadding_tokens": 6306, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3680786879999687, "nonpadding_tokens": 9365, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37027544000000034, "nonpadding_tokens": 6493, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3669118669999989, "nonpadding_tokens": 2210, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003697129999409299, "prefetch": {"data_wait_seconds": 0.020488293001292845, "consumed_batches": 2180, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2730, "tokens": 199628680, "loss_tokens": 27332, "loss": 2.2136853950954927, "lr": 0.0002991341724202597, "optimizer_lrs": [0.01994227816135065, 0.0002991341724202597], "gradient_norm": 0.4823712408542633, "seconds": 0.24133336700003838, "nonpadding_tokens_per_second": 113254.12784712714, "ranks": [{"compute_seconds": 0.23874328099998365, "nonpadding_tokens": 2257, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2383217679999916, "nonpadding_tokens": 2888, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23718564900002548, "nonpadding_tokens": 5554, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23654224799997792, "nonpadding_tokens": 3083, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23428346499997588, "nonpadding_tokens": 3446, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23718287399992732, "nonpadding_tokens": 1040, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23417299400000502, "nonpadding_tokens": 2799, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23900159300001178, "nonpadding_tokens": 6265, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003728389999650972, "prefetch": {"data_wait_seconds": 0.020496797001285927, "consumed_batches": 2181, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2731, "tokens": 199686496, "loss_tokens": 57816, "loss": 2.194457146378165, "lr": 0.0002991336155701771, "optimizer_lrs": [0.01994224103801181, 0.0002991336155701771], "gradient_norm": 0.4234378933906555, "seconds": 0.41529420500000924, "nonpadding_tokens_per_second": 139216.96788424658, "ranks": [{"compute_seconds": 0.40954749799993806, "nonpadding_tokens": 4700, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4130043149999665, "nonpadding_tokens": 2796, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.40878697799996644, "nonpadding_tokens": 9630, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.40931630099998983, "nonpadding_tokens": 3021, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.40805579499999567, "nonpadding_tokens": 11051, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4101843019999478, "nonpadding_tokens": 9354, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.41173334300003717, "nonpadding_tokens": 4011, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.40799840800002585, "nonpadding_tokens": 13253, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003626879999956145, "prefetch": {"data_wait_seconds": 0.020505374001302812, "consumed_batches": 2182, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2732, "tokens": 199751119, "loss_tokens": 64623, "loss": 2.325111850656887, "lr": 0.00029913299294784864, "optimizer_lrs": [0.019942199529856576, 0.00029913299294784864], "gradient_norm": 0.4164392650127411, "seconds": 0.22944509199999175, "nonpadding_tokens_per_second": 281649.0840431763, "ranks": [{"compute_seconds": 0.2268975470000214, "nonpadding_tokens": 3967, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22342080099997474, "nonpadding_tokens": 9740, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22298062400000163, "nonpadding_tokens": 7757, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22329195700001492, "nonpadding_tokens": 11175, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22687714799997138, "nonpadding_tokens": 4358, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22342715799993584, "nonpadding_tokens": 6668, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22171478699999625, "nonpadding_tokens": 7227, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22250072400004228, "nonpadding_tokens": 13731, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040286199998718075, "prefetch": {"data_wait_seconds": 0.020517061001285697, "consumed_batches": 2183, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2733, "tokens": 199842683, "loss_tokens": 91564, "loss": 2.223246684013368, "lr": 0.00029913211037574367, "optimizer_lrs": [0.019942140691716247, 0.00029913211037574367], "gradient_norm": 0.39603888988494873, "seconds": 0.24602911699992092, "nonpadding_tokens_per_second": 372167.33172289294, "ranks": [{"compute_seconds": 0.24295411200000672, "nonpadding_tokens": 8621, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24014241899999433, "nonpadding_tokens": 10485, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24078971100004765, "nonpadding_tokens": 3023, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24332899299997734, "nonpadding_tokens": 24887, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24361498199994003, "nonpadding_tokens": 10297, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24263264899991555, "nonpadding_tokens": 11850, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23961806900001648, "nonpadding_tokens": 8805, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24017568499994013, "nonpadding_tokens": 13596, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003972869999415707, "prefetch": {"data_wait_seconds": 0.02052616700127885, "consumed_batches": 2184, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2734, "tokens": 199901932, "loss_tokens": 59249, "loss": 2.3978693205792503, "lr": 0.0002991315390447464, "optimizer_lrs": [0.019942102602983094, 0.0002991315390447464], "gradient_norm": 0.3707875907421112, "seconds": 0.3119577310000068, "nonpadding_tokens_per_second": 189926.37178784554, "ranks": [{"compute_seconds": 0.3089174389999698, "nonpadding_tokens": 8540, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3095940100000689, "nonpadding_tokens": 9023, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30834466899989366, "nonpadding_tokens": 5568, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3089676959999679, "nonpadding_tokens": 7962, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3092726180000227, "nonpadding_tokens": 11937, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3079946499999551, "nonpadding_tokens": 6316, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3054434019999235, "nonpadding_tokens": 4593, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3096530090000442, "nonpadding_tokens": 5310, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003895229999670846, "prefetch": {"data_wait_seconds": 0.020535816001213902, "consumed_batches": 2185, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2735, "tokens": 200032580, "loss_tokens": 130648, "loss": 2.5517686359530956, "lr": 0.0002991302785596011, "optimizer_lrs": [0.019942018570640076, 0.0002991302785596011], "gradient_norm": 0.5063401460647583, "seconds": 0.2363603769999827, "nonpadding_tokens_per_second": 552749.1606599087, "ranks": [{"compute_seconds": 0.23197614699995484, "nonpadding_tokens": 2469, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23387021000007735, "nonpadding_tokens": 20584, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23004188900006284, "nonpadding_tokens": 6419, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23128296599998066, "nonpadding_tokens": 18027, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23254113200005122, "nonpadding_tokens": 13803, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23222517099998186, "nonpadding_tokens": 25316, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22906494599999405, "nonpadding_tokens": 13143, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23312457000008635, "nonpadding_tokens": 30887, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039480600003116706, "prefetch": {"data_wait_seconds": 0.02054323200123065, "consumed_batches": 2186, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2736, "tokens": 200124020, "loss_tokens": 91440, "loss": 2.1324686953193353, "lr": 0.0002991293958092111, "optimizer_lrs": [0.019941959720614078, 0.0002991293958092111], "gradient_norm": 0.47302141785621643, "seconds": 0.3537143730000025, "nonpadding_tokens_per_second": 258513.66803236844, "ranks": [{"compute_seconds": 0.35052876199995353, "nonpadding_tokens": 18169, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35129777000008744, "nonpadding_tokens": 29005, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35063708900008805, "nonpadding_tokens": 8095, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35125951200006966, "nonpadding_tokens": 4443, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.347641952999993, "nonpadding_tokens": 12747, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35057840599995416, "nonpadding_tokens": 6226, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34708141099997647, "nonpadding_tokens": 9103, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3484870360000514, "nonpadding_tokens": 3652, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043067299998256203, "prefetch": {"data_wait_seconds": 0.02055227700118678, "consumed_batches": 2187, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2737, "tokens": 200173135, "loss_tokens": 49115, "loss": 2.031959431945434, "lr": 0.0002991289214748632, "optimizer_lrs": [0.019941928098324218, 0.0002991289214748632], "gradient_norm": 0.45426180958747864, "seconds": 0.22187211000004936, "nonpadding_tokens_per_second": 221366.26365517086, "ranks": [{"compute_seconds": 0.2110973250000825, "nonpadding_tokens": 4166, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21555033200002072, "nonpadding_tokens": 8826, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21626031699997839, "nonpadding_tokens": 2890, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21869252199996936, "nonpadding_tokens": 13422, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2153516949999812, "nonpadding_tokens": 10190, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21799652899994726, "nonpadding_tokens": 4126, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21919748900006653, "nonpadding_tokens": 4067, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2195079740000665, "nonpadding_tokens": 1428, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000401680000095439, "prefetch": {"data_wait_seconds": 0.02056092500124862, "consumed_batches": 2188, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2738, "tokens": 200224995, "loss_tokens": 51860, "loss": 2.0814074250385652, "lr": 0.0002991284204905703, "optimizer_lrs": [0.019941894699371354, 0.0002991284204905703], "gradient_norm": 0.658145546913147, "seconds": 0.2836742399999821, "nonpadding_tokens_per_second": 182815.33071174624, "ranks": [{"compute_seconds": 0.2801773039999489, "nonpadding_tokens": 1226, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2813597199999549, "nonpadding_tokens": 11277, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2780160980000801, "nonpadding_tokens": 11643, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2807822380000289, "nonpadding_tokens": 10932, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.28115744199999426, "nonpadding_tokens": 3024, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2797700469999427, "nonpadding_tokens": 2808, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27719871299996157, "nonpadding_tokens": 2284, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2813909660000036, "nonpadding_tokens": 8666, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003529020000314631, "prefetch": {"data_wait_seconds": 0.020569630001276096, "consumed_batches": 2189, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2739, "tokens": 200276716, "loss_tokens": 51721, "loss": 2.4924957947448814, "lr": 0.0002991279207060808, "optimizer_lrs": [0.01994186138040539, 0.0002991279207060808], "gradient_norm": 0.4233427345752716, "seconds": 0.2710719319999271, "nonpadding_tokens_per_second": 190801.75368364554, "ranks": [{"compute_seconds": 0.26810690199999954, "nonpadding_tokens": 10791, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26862714399999277, "nonpadding_tokens": 7745, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2652633389999437, "nonpadding_tokens": 12753, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2668736019999187, "nonpadding_tokens": 3752, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26872861499998635, "nonpadding_tokens": 1729, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26794266999991123, "nonpadding_tokens": 6121, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2649467820000382, "nonpadding_tokens": 3397, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26503153800001655, "nonpadding_tokens": 5433, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033652900003744435, "prefetch": {"data_wait_seconds": 0.020577219001282288, "consumed_batches": 2190, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2740, "tokens": 200345613, "loss_tokens": 68897, "loss": 2.066106606601158, "lr": 0.0002991272547266299, "optimizer_lrs": [0.01994181698177533, 0.0002991272547266299], "gradient_norm": 0.5731107592582703, "seconds": 0.28267470299999786, "nonpadding_tokens_per_second": 243732.45737522017, "ranks": [{"compute_seconds": 0.2803147899999203, "nonpadding_tokens": 13303, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27904498700002023, "nonpadding_tokens": 2808, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27624520500000926, "nonpadding_tokens": 5000, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27701969999998255, "nonpadding_tokens": 7394, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2789998179999884, "nonpadding_tokens": 6492, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27648671899999044, "nonpadding_tokens": 9620, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27541498199991565, "nonpadding_tokens": 11570, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2755999349999456, "nonpadding_tokens": 12710, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034525000000940054, "prefetch": {"data_wait_seconds": 0.020585560001293857, "consumed_batches": 2191, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2741, "tokens": 200390701, "loss_tokens": 45088, "loss": 2.6800635008206175, "lr": 0.00029912681875508395, "optimizer_lrs": [0.0199417879170056, 0.00029912681875508395], "gradient_norm": 0.44533997774124146, "seconds": 0.32340641700000106, "nonpadding_tokens_per_second": 139415.9102291401, "ranks": [{"compute_seconds": 0.3210767159999932, "nonpadding_tokens": 14399, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32055380099996, "nonpadding_tokens": 2146, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3196727890000375, "nonpadding_tokens": 7830, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31829287799996564, "nonpadding_tokens": 5465, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31652617600002486, "nonpadding_tokens": 4024, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3197153920000346, "nonpadding_tokens": 4594, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3205060790000971, "nonpadding_tokens": 3664, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31706008399999064, "nonpadding_tokens": 2966, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003826800000297226, "prefetch": {"data_wait_seconds": 0.0205961200012581, "consumed_batches": 2192, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2742, "tokens": 200458461, "loss_tokens": 67760, "loss": 2.2959560858175916, "lr": 0.00029912616335601164, "optimizer_lrs": [0.019941744223734112, 0.00029912616335601164], "gradient_norm": 0.4062323272228241, "seconds": 0.26601681399995414, "nonpadding_tokens_per_second": 254720.7410732003, "ranks": [{"compute_seconds": 0.2636343849999321, "nonpadding_tokens": 12854, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26354786800004604, "nonpadding_tokens": 5292, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26307929799997964, "nonpadding_tokens": 9851, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2633958110000094, "nonpadding_tokens": 14265, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.260151063999956, "nonpadding_tokens": 1998, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2624347180000086, "nonpadding_tokens": 4322, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26014836700005617, "nonpadding_tokens": 6888, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25999723100005667, "nonpadding_tokens": 12290, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004262180000296212, "prefetch": {"data_wait_seconds": 0.020605816001264543, "consumed_batches": 2193, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2743, "tokens": 200508219, "loss_tokens": 49758, "loss": 2.05426922555167, "lr": 0.0002991256819226929, "optimizer_lrs": [0.01994171212817953, 0.0002991256819226929], "gradient_norm": 0.4709416925907135, "seconds": 0.25029044099994735, "nonpadding_tokens_per_second": 198801.04010848128, "ranks": [{"compute_seconds": 0.24765070099999775, "nonpadding_tokens": 4434, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24362094600007822, "nonpadding_tokens": 10660, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24347660600005838, "nonpadding_tokens": 3341, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2441654420000532, "nonpadding_tokens": 7129, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24239871099996435, "nonpadding_tokens": 3945, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2449448489999213, "nonpadding_tokens": 5731, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24772100699999555, "nonpadding_tokens": 6600, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24259565800002747, "nonpadding_tokens": 7918, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038751800002501113, "prefetch": {"data_wait_seconds": 0.020613913001241713, "consumed_batches": 2194, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2744, "tokens": 200563076, "loss_tokens": 54857, "loss": 2.2904638708369034, "lr": 0.00029912515100086, "optimizer_lrs": [0.01994167673339067, 0.00029912515100086], "gradient_norm": 0.41566020250320435, "seconds": 0.26732169499996417, "nonpadding_tokens_per_second": 205209.68191529444, "ranks": [{"compute_seconds": 0.2638228710000021, "nonpadding_tokens": 2778, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2636182840000174, "nonpadding_tokens": 4252, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26067271400006575, "nonpadding_tokens": 7506, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26428036999993765, "nonpadding_tokens": 7544, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2596231820000412, "nonpadding_tokens": 5995, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26268632599999364, "nonpadding_tokens": 7092, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2599351349999779, "nonpadding_tokens": 6806, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26482227899998634, "nonpadding_tokens": 12884, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004317469999932655, "prefetch": {"data_wait_seconds": 0.02062445600127205, "consumed_batches": 2195, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2745, "tokens": 200619225, "loss_tokens": 56149, "loss": 2.545715417905929, "lr": 0.00029912460740833425, "optimizer_lrs": [0.01994164049388895, 0.00029912460740833425], "gradient_norm": 0.4384956359863281, "seconds": 0.3023034269999698, "nonpadding_tokens_per_second": 185737.22619428198, "ranks": [{"compute_seconds": 0.2446781529999953, "nonpadding_tokens": 3647, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24445329099989976, "nonpadding_tokens": 6638, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24178771500010043, "nonpadding_tokens": 13444, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2424575630000163, "nonpadding_tokens": 2901, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2468317409999372, "nonpadding_tokens": 4067, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24410142299996096, "nonpadding_tokens": 6866, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2405007160000423, "nonpadding_tokens": 3529, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2418011549999619, "nonpadding_tokens": 15057, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0072540550000894655, "prefetch": {"data_wait_seconds": 0.020633516001296925, "consumed_batches": 2196, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2746, "tokens": 200679589, "loss_tokens": 60364, "loss": 2.671601658273143, "lr": 0.000299124022821635, "optimizer_lrs": [0.019941601521442334, 0.000299124022821635], "gradient_norm": 0.46893420815467834, "seconds": 0.6463990299999978, "nonpadding_tokens_per_second": 93385.0411254488, "ranks": [{"compute_seconds": 0.6370668450000494, "nonpadding_tokens": 10622, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.6394927530000132, "nonpadding_tokens": 8853, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.6429023229999302, "nonpadding_tokens": 4380, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.6430486030000111, "nonpadding_tokens": 3142, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.635774303000062, "nonpadding_tokens": 4613, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.6384642380000969, "nonpadding_tokens": 19325, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.6358462450000388, "nonpadding_tokens": 5325, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.6439245650000203, "nonpadding_tokens": 4104, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000399652999931277, "prefetch": {"data_wait_seconds": 0.020643037001221387, "consumed_batches": 2197, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2747, "tokens": 200731357, "loss_tokens": 51768, "loss": 2.0111190189885644, "lr": 0.00029912352132677276, "optimizer_lrs": [0.019941568088451517, 0.00029912352132677276], "gradient_norm": 0.43683940172195435, "seconds": 0.25878597999997055, "nonpadding_tokens_per_second": 200041.74878409522, "ranks": [{"compute_seconds": 0.25545000199997503, "nonpadding_tokens": 2428, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2557749000000058, "nonpadding_tokens": 6451, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25513336100004835, "nonpadding_tokens": 5549, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2553596090000383, "nonpadding_tokens": 3275, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2517424050000727, "nonpadding_tokens": 12540, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25415277599995534, "nonpadding_tokens": 2855, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25611409800001184, "nonpadding_tokens": 8619, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25635646900002484, "nonpadding_tokens": 10051, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000382056000034936, "prefetch": {"data_wait_seconds": 0.020651723001222422, "consumed_batches": 2198, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2748, "tokens": 200795662, "loss_tokens": 64305, "loss": 2.195947535183889, "lr": 0.00029912289818237155, "optimizer_lrs": [0.019941526545491438, 0.00029912289818237155], "gradient_norm": 0.38975077867507935, "seconds": 0.2981908949999479, "nonpadding_tokens_per_second": 215650.4476771876, "ranks": [{"compute_seconds": 0.2957974839999906, "nonpadding_tokens": 6430, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29464082499998767, "nonpadding_tokens": 5900, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29235619400003543, "nonpadding_tokens": 4519, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2949657119999074, "nonpadding_tokens": 19624, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2955745530000513, "nonpadding_tokens": 14041, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29341444400006367, "nonpadding_tokens": 5522, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2910405320000109, "nonpadding_tokens": 3467, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2913387309999962, "nonpadding_tokens": 4802, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035960100001375395, "prefetch": {"data_wait_seconds": 0.020660598001157382, "consumed_batches": 2199, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2749, "tokens": 200873772, "loss_tokens": 78110, "loss": 1.9398606532454232, "lr": 0.00029912214096442836, "optimizer_lrs": [0.019941476064295225, 0.00029912214096442836], "gradient_norm": 0.5451249480247498, "seconds": 0.326235294000071, "nonpadding_tokens_per_second": 239428.41696331912, "ranks": [{"compute_seconds": 0.32370167199997013, "nonpadding_tokens": 12156, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3188151450000305, "nonpadding_tokens": 17844, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3181099500000073, "nonpadding_tokens": 17465, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31919346600000154, "nonpadding_tokens": 5192, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3226212410000926, "nonpadding_tokens": 7854, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.319834604000107, "nonpadding_tokens": 3650, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3172531859999026, "nonpadding_tokens": 6085, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3182959900000242, "nonpadding_tokens": 7864, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005014280000068538, "prefetch": {"data_wait_seconds": 0.020668045001187352, "consumed_batches": 2200, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2750, "tokens": 200940320, "loss_tokens": 66548, "loss": 2.1352217102692794, "lr": 0.0002991214955745, "optimizer_lrs": [0.019941433038300004, 0.0002991214955745], "gradient_norm": 0.6170068979263306, "seconds": 0.33251501699999153, "nonpadding_tokens_per_second": 200135.32200863486, "ranks": [{"compute_seconds": 0.32779189300003964, "nonpadding_tokens": 16187, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3292745899999545, "nonpadding_tokens": 5048, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3267299720000665, "nonpadding_tokens": 2341, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32938571200008937, "nonpadding_tokens": 3950, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3254033949999666, "nonpadding_tokens": 3217, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3269183710000334, "nonpadding_tokens": 9895, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32985264199999165, "nonpadding_tokens": 8360, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3301325679999536, "nonpadding_tokens": 17550, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004063560000986399, "prefetch": {"data_wait_seconds": 0.020677400001162027, "consumed_batches": 2201, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2751, "tokens": 201001853, "loss_tokens": 61533, "loss": 2.3294075942989942, "lr": 0.0002991208986102707, "optimizer_lrs": [0.019941393240684717, 0.0002991208986102707], "gradient_norm": 0.47469615936279297, "seconds": 0.3137127730000202, "nonpadding_tokens_per_second": 196144.38842117542, "ranks": [{"compute_seconds": 0.3037117119999948, "nonpadding_tokens": 3704, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30704067499993926, "nonpadding_tokens": 4222, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.305775165, "nonpadding_tokens": 7877, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3065215309999303, "nonpadding_tokens": 4815, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3049861709999959, "nonpadding_tokens": 12764, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30794163500002014, "nonpadding_tokens": 6705, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3054264670000748, "nonpadding_tokens": 10055, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31134804299995267, "nonpadding_tokens": 11391, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034874899995429587, "prefetch": {"data_wait_seconds": 0.020685919001110165, "consumed_batches": 2202, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2752, "tokens": 201073545, "loss_tokens": 71692, "loss": 1.9553378079144117, "lr": 0.0002991202028333043, "optimizer_lrs": [0.01994134685555362, 0.0002991202028333043], "gradient_norm": 0.39065632224082947, "seconds": 0.4138047829999323, "nonpadding_tokens_per_second": 173250.77656246358, "ranks": [{"compute_seconds": 0.4105277170000363, "nonpadding_tokens": 16271, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4067581169999812, "nonpadding_tokens": 11010, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.408408326999961, "nonpadding_tokens": 12634, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4083446309999772, "nonpadding_tokens": 2073, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.41136730100004115, "nonpadding_tokens": 2478, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.40792250999993485, "nonpadding_tokens": 6740, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.40671230500004185, "nonpadding_tokens": 6134, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4114173920000894, "nonpadding_tokens": 14352, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003377790000058667, "prefetch": {"data_wait_seconds": 0.020693826001163984, "consumed_batches": 2203, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2753, "tokens": 201154752, "loss_tokens": 81207, "loss": 2.1923733483566687, "lr": 0.00029911941438109, "optimizer_lrs": [0.01994129429207267, 0.00029911941438109], "gradient_norm": 0.39605844020843506, "seconds": 0.3424869589999844, "nonpadding_tokens_per_second": 237109.75809739868, "ranks": [{"compute_seconds": 0.33692978999999923, "nonpadding_tokens": 8788, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33659311500002786, "nonpadding_tokens": 6091, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3393244070000492, "nonpadding_tokens": 14080, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33958171899996614, "nonpadding_tokens": 3915, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3400264160000006, "nonpadding_tokens": 10337, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3380915150000874, "nonpadding_tokens": 19323, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3401138589999846, "nonpadding_tokens": 10426, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33544043499989584, "nonpadding_tokens": 8247, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042332600003192056, "prefetch": {"data_wait_seconds": 0.020703579001136063, "consumed_batches": 2204, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2754, "tokens": 201208974, "loss_tokens": 54222, "loss": 2.4649787563166243, "lr": 0.00029911888773474023, "optimizer_lrs": [0.019941259182316016, 0.00029911888773474023], "gradient_norm": 0.45958566665649414, "seconds": 0.2981017239999346, "nonpadding_tokens_per_second": 181890.9306274656, "ranks": [{"compute_seconds": 0.2950096309999708, "nonpadding_tokens": 12351, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2913981460000059, "nonpadding_tokens": 8630, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2924717820000069, "nonpadding_tokens": 6748, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29577627399999074, "nonpadding_tokens": 9119, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2910850020000453, "nonpadding_tokens": 3776, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2934189500000457, "nonpadding_tokens": 5413, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2910330199999862, "nonpadding_tokens": 3906, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2909814209999695, "nonpadding_tokens": 4279, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003577160000531876, "prefetch": {"data_wait_seconds": 0.02071338300113723, "consumed_batches": 2205, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2755, "tokens": 201296451, "loss_tokens": 87477, "loss": 2.393892644638019, "lr": 0.00029911803775909936, "optimizer_lrs": [0.019941202517273292, 0.00029911803775909936], "gradient_norm": 0.4225466251373291, "seconds": 0.3429201719999355, "nonpadding_tokens_per_second": 255094.35472934632, "ranks": [{"compute_seconds": 0.3397615940000378, "nonpadding_tokens": 7640, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3403141119999873, "nonpadding_tokens": 10368, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3396482959999503, "nonpadding_tokens": 3630, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3397648410000329, "nonpadding_tokens": 18563, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33641958199996225, "nonpadding_tokens": 8552, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3398193699999865, "nonpadding_tokens": 12545, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.340558255000019, "nonpadding_tokens": 20075, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33661456499999076, "nonpadding_tokens": 6104, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000396477999970557, "prefetch": {"data_wait_seconds": 0.020722822001133068, "consumed_batches": 2206, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2756, "tokens": 201367587, "loss_tokens": 71136, "loss": 2.2660304733187133, "lr": 0.0002991173462606772, "optimizer_lrs": [0.01994115641737848, 0.0002991173462606772], "gradient_norm": 0.4516248404979706, "seconds": 0.30682618799994543, "nonpadding_tokens_per_second": 231844.61686175448, "ranks": [{"compute_seconds": 0.303795507000018, "nonpadding_tokens": 6528, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3040043539999715, "nonpadding_tokens": 7963, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3031690550000121, "nonpadding_tokens": 9139, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3012338929999032, "nonpadding_tokens": 14494, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3043476880000071, "nonpadding_tokens": 8508, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30126837700004216, "nonpadding_tokens": 4542, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2995604249999815, "nonpadding_tokens": 9776, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29995934700002636, "nonpadding_tokens": 10186, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004009479999922405, "prefetch": {"data_wait_seconds": 0.020732554001099857, "consumed_batches": 2207, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2757, "tokens": 201469380, "loss_tokens": 101793, "loss": 2.1450834966549763, "lr": 0.00029911635628209875, "optimizer_lrs": [0.019941090418806585, 0.00029911635628209875], "gradient_norm": 0.43986350297927856, "seconds": 0.28481367999995655, "nonpadding_tokens_per_second": 357402.07422626443, "ranks": [{"compute_seconds": 0.2822921290000977, "nonpadding_tokens": 8187, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2819245809999984, "nonpadding_tokens": 11489, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27893047599991405, "nonpadding_tokens": 7067, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27897054099992147, "nonpadding_tokens": 9690, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.277583060999973, "nonpadding_tokens": 10125, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2787575720000177, "nonpadding_tokens": 18087, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28171852600007696, "nonpadding_tokens": 17629, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2824484999999868, "nonpadding_tokens": 19519, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004022489999897516, "prefetch": {"data_wait_seconds": 0.02074255500117772, "consumed_batches": 2208, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2758, "tokens": 201538056, "loss_tokens": 68676, "loss": 2.3437713866561825, "lr": 0.00029911568806747336, "optimizer_lrs": [0.01994104587116489, 0.00029911568806747336], "gradient_norm": 0.4502682387828827, "seconds": 0.29513436000002, "nonpadding_tokens_per_second": 232694.017734822, "ranks": [{"compute_seconds": 0.2335435420000067, "nonpadding_tokens": 5279, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24378414600005271, "nonpadding_tokens": 6869, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24457173300004342, "nonpadding_tokens": 3910, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2449087549999831, "nonpadding_tokens": 7535, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24311050699998304, "nonpadding_tokens": 8963, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2448125490000166, "nonpadding_tokens": 17326, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24733070100000987, "nonpadding_tokens": 7919, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24808892599992305, "nonpadding_tokens": 10875, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.005478354999922885, "prefetch": {"data_wait_seconds": 0.02075086600116549, "consumed_batches": 2209, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2759, "tokens": 201630790, "loss_tokens": 92734, "loss": 2.482485017631074, "lr": 0.0002991147853700603, "optimizer_lrs": [0.019940985691337353, 0.0002991147853700603], "gradient_norm": 0.4855611324310303, "seconds": 0.32478174799996395, "nonpadding_tokens_per_second": 285527.12882132246, "ranks": [{"compute_seconds": 0.32177950899995267, "nonpadding_tokens": 17664, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31941115699999045, "nonpadding_tokens": 4681, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32246962800002166, "nonpadding_tokens": 11595, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32029474699993443, "nonpadding_tokens": 3945, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3189428040000166, "nonpadding_tokens": 10125, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3218717669999478, "nonpadding_tokens": 9650, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3190687109999999, "nonpadding_tokens": 10247, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3202630279999994, "nonpadding_tokens": 24827, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033601099994484684, "prefetch": {"data_wait_seconds": 0.020759083001124168, "consumed_batches": 2210, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2760, "tokens": 201719082, "loss_tokens": 88292, "loss": 2.3757430953540526, "lr": 0.00029911392548572586, "optimizer_lrs": [0.01994092836571506, 0.00029911392548572586], "gradient_norm": 0.47831130027770996, "seconds": 0.34428524099996594, "nonpadding_tokens_per_second": 256450.14507028705, "ranks": [{"compute_seconds": 0.33995083199999954, "nonpadding_tokens": 9596, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3419896099999278, "nonpadding_tokens": 16231, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33959838699990996, "nonpadding_tokens": 11786, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3397962290000578, "nonpadding_tokens": 10385, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.34141641100006836, "nonpadding_tokens": 3475, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3386079949999612, "nonpadding_tokens": 7001, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3380802820000781, "nonpadding_tokens": 16666, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3384810679999646, "nonpadding_tokens": 13152, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035390499999721214, "prefetch": {"data_wait_seconds": 0.020769105001093635, "consumed_batches": 2211, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2761, "tokens": 201797403, "loss_tokens": 78321, "loss": 2.0956438487123505, "lr": 0.00029911316236169076, "optimizer_lrs": [0.019940877490779385, 0.00029911316236169076], "gradient_norm": 0.3515464663505554, "seconds": 0.2917694179999444, "nonpadding_tokens_per_second": 268434.5759637322, "ranks": [{"compute_seconds": 0.2887948410000263, "nonpadding_tokens": 4420, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28942104299994753, "nonpadding_tokens": 21763, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28568754699995225, "nonpadding_tokens": 1299, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28673974600008023, "nonpadding_tokens": 25257, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2850762319999376, "nonpadding_tokens": 4407, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2876824069999202, "nonpadding_tokens": 11186, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2889752129999579, "nonpadding_tokens": 2076, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28538566899999296, "nonpadding_tokens": 7913, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038790500002505723, "prefetch": {"data_wait_seconds": 0.020779049001092176, "consumed_batches": 2212, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2762, "tokens": 201863856, "loss_tokens": 66453, "loss": 1.9459271261267361, "lr": 0.000299112514617288, "optimizer_lrs": [0.0199408343078192, 0.000299112514617288], "gradient_norm": 0.4704231917858124, "seconds": 0.29860326200002874, "nonpadding_tokens_per_second": 222546.12878272444, "ranks": [{"compute_seconds": 0.29571649299998626, "nonpadding_tokens": 11292, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29634542200005853, "nonpadding_tokens": 12945, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2931839849999278, "nonpadding_tokens": 10174, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2935701019999897, "nonpadding_tokens": 2214, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29222007099997427, "nonpadding_tokens": 9791, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2935379190000731, "nonpadding_tokens": 4590, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2958649070000092, "nonpadding_tokens": 11077, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29296771199994964, "nonpadding_tokens": 4370, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003894879999961631, "prefetch": {"data_wait_seconds": 0.020795332001128486, "consumed_batches": 2213, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2763, "tokens": 201929657, "loss_tokens": 65801, "loss": 1.9219766321940397, "lr": 0.0002991118729959625, "optimizer_lrs": [0.01994079153306417, 0.0002991118729959625], "gradient_norm": 0.420019268989563, "seconds": 0.32275752200007446, "nonpadding_tokens_per_second": 203871.31364822172, "ranks": [{"compute_seconds": 0.32003703999998834, "nonpadding_tokens": 7927, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.319966620999935, "nonpadding_tokens": 9700, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3189056900000651, "nonpadding_tokens": 3636, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3173800360000314, "nonpadding_tokens": 1330, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3157223339999291, "nonpadding_tokens": 10912, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3190345599999773, "nonpadding_tokens": 17912, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3156213490000255, "nonpadding_tokens": 9320, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3200500010000269, "nonpadding_tokens": 5064, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004053789999716173, "prefetch": {"data_wait_seconds": 0.020805224001108513, "consumed_batches": 2214, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2764, "tokens": 202050521, "loss_tokens": 120864, "loss": 2.4963026624966904, "lr": 0.0002991106938568029, "optimizer_lrs": [0.019940712923786862, 0.0002991106938568029], "gradient_norm": 0.54316246509552, "seconds": 0.38621685999999045, "nonpadding_tokens_per_second": 312943.3551916998, "ranks": [{"compute_seconds": 0.3818323309999414, "nonpadding_tokens": 18051, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.383871863999957, "nonpadding_tokens": 12987, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.38239724499999284, "nonpadding_tokens": 7981, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3836652889999641, "nonpadding_tokens": 17122, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.38383596899996064, "nonpadding_tokens": 24423, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38246323500004564, "nonpadding_tokens": 19156, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3833391279999887, "nonpadding_tokens": 19473, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.38009507599997505, "nonpadding_tokens": 1671, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003606059999583522, "prefetch": {"data_wait_seconds": 0.020813712001157114, "consumed_batches": 2215, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2765, "tokens": 202131534, "loss_tokens": 81013, "loss": 2.1636392461703675, "lr": 0.00029910990306426224, "optimizer_lrs": [0.01994066020428415, 0.00029910990306426224], "gradient_norm": 0.39371296763420105, "seconds": 0.30000550699992345, "nonpadding_tokens_per_second": 270038.37632894074, "ranks": [{"compute_seconds": 0.2964136159999953, "nonpadding_tokens": 11933, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29776155900003687, "nonpadding_tokens": 7598, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29699045100005605, "nonpadding_tokens": 18196, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2971948050000037, "nonpadding_tokens": 6104, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29407978000006096, "nonpadding_tokens": 12518, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29624840799999674, "nonpadding_tokens": 1181, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2974971280000318, "nonpadding_tokens": 9937, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29406854900003054, "nonpadding_tokens": 13546, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003476339999224365, "prefetch": {"data_wait_seconds": 0.020821949001131657, "consumed_batches": 2216, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2766, "tokens": 202189063, "loss_tokens": 57529, "loss": 2.4403507361504633, "lr": 0.0002991093412935037, "optimizer_lrs": [0.019940622752900247, 0.0002991093412935037], "gradient_norm": 0.4618355929851532, "seconds": 0.2677437419999933, "nonpadding_tokens_per_second": 214865.89964818466, "ranks": [{"compute_seconds": 0.26487187899999753, "nonpadding_tokens": 2371, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26175697299993317, "nonpadding_tokens": 4928, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.262006263000103, "nonpadding_tokens": 17684, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26287300399997093, "nonpadding_tokens": 13640, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2650860380000495, "nonpadding_tokens": 7965, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2636465010000393, "nonpadding_tokens": 3279, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26126199800000904, "nonpadding_tokens": 3194, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26544336699998894, "nonpadding_tokens": 4468, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003706659999807016, "prefetch": {"data_wait_seconds": 0.02083033000106127, "consumed_batches": 2217, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2767, "tokens": 202269828, "loss_tokens": 80765, "loss": 2.193332314430756, "lr": 0.00029910855232512045, "optimizer_lrs": [0.019940570155008032, 0.00029910855232512045], "gradient_norm": 0.46082088351249695, "seconds": 0.34397042600005534, "nonpadding_tokens_per_second": 234802.1629045138, "ranks": [{"compute_seconds": 0.34005228800003806, "nonpadding_tokens": 11910, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33772386699990875, "nonpadding_tokens": 17495, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3406714010000087, "nonpadding_tokens": 24935, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3416537379999909, "nonpadding_tokens": 10220, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.33791554200001883, "nonpadding_tokens": 4277, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34065841899996485, "nonpadding_tokens": 4639, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33736186800001633, "nonpadding_tokens": 5609, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.338108831999989, "nonpadding_tokens": 1680, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003461810000544574, "prefetch": {"data_wait_seconds": 0.020839808001099414, "consumed_batches": 2218, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2768, "tokens": 202339567, "loss_tokens": 69739, "loss": 2.237405047749466, "lr": 0.0002991078707862456, "optimizer_lrs": [0.01994052471908304, 0.0002991078707862456], "gradient_norm": 0.4367138147354126, "seconds": 0.2796905110000125, "nonpadding_tokens_per_second": 249343.46092276572, "ranks": [{"compute_seconds": 0.27744648600003075, "nonpadding_tokens": 1760, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27379871400000866, "nonpadding_tokens": 3335, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2762459549999221, "nonpadding_tokens": 6532, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27506869700005154, "nonpadding_tokens": 10268, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2732972610000388, "nonpadding_tokens": 5186, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27501484399999754, "nonpadding_tokens": 12224, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2732061680000015, "nonpadding_tokens": 3686, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2738877250000087, "nonpadding_tokens": 26748, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003626010000061797, "prefetch": {"data_wait_seconds": 0.02084771900111093, "consumed_batches": 2219, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2769, "tokens": 202433312, "loss_tokens": 93745, "loss": 2.423515920849112, "lr": 0.00029910695423477826, "optimizer_lrs": [0.019940463615651886, 0.00029910695423477826], "gradient_norm": 0.35910552740097046, "seconds": 0.26439090300004864, "nonpadding_tokens_per_second": 354569.6880500565, "ranks": [{"compute_seconds": 0.2616842850000012, "nonpadding_tokens": 7343, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2581565140000066, "nonpadding_tokens": 11743, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25892730600003233, "nonpadding_tokens": 19642, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25946784199993544, "nonpadding_tokens": 12018, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2577922249998892, "nonpadding_tokens": 7044, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25908594600002743, "nonpadding_tokens": 7581, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25765356700003395, "nonpadding_tokens": 12745, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2620341570000164, "nonpadding_tokens": 15629, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004034109999793145, "prefetch": {"data_wait_seconds": 0.02085534900113828, "consumed_batches": 2220, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2770, "tokens": 202513455, "loss_tokens": 80143, "loss": 2.3148900636986385, "lr": 0.00029910617029914717, "optimizer_lrs": [0.01994041135327648, 0.00029910617029914717], "gradient_norm": 0.4049268364906311, "seconds": 0.486175641999921, "nonpadding_tokens_per_second": 164843.71711903455, "ranks": [{"compute_seconds": 0.4821834009999293, "nonpadding_tokens": 17631, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4816541760000064, "nonpadding_tokens": 6852, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4785688020000407, "nonpadding_tokens": 10958, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.47892614299996694, "nonpadding_tokens": 5285, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.4837296809999998, "nonpadding_tokens": 2078, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4806539319999956, "nonpadding_tokens": 12130, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.47758735599995816, "nonpadding_tokens": 20209, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4781944480000675, "nonpadding_tokens": 5000, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003873709999879793, "prefetch": {"data_wait_seconds": 0.020865275001099803, "consumed_batches": 2221, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2771, "tokens": 202595819, "loss_tokens": 82364, "loss": 2.219647312539459, "lr": 0.0002991053642811716, "optimizer_lrs": [0.019940357618744776, 0.0002991053642811716], "gradient_norm": 0.44402265548706055, "seconds": 0.24849905300004593, "nonpadding_tokens_per_second": 331445.9311037486, "ranks": [{"compute_seconds": 0.2436250819999941, "nonpadding_tokens": 3473, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24614262500006134, "nonpadding_tokens": 4786, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24559446499995374, "nonpadding_tokens": 14777, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2454531280000083, "nonpadding_tokens": 17876, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24238812700002654, "nonpadding_tokens": 20292, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24335101799999848, "nonpadding_tokens": 11266, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24586484600001768, "nonpadding_tokens": 6179, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24289477199999965, "nonpadding_tokens": 3715, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040504300000065996, "prefetch": {"data_wait_seconds": 0.020875307001077203, "consumed_batches": 2222, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2772, "tokens": 202674870, "loss_tokens": 79051, "loss": 1.8832393407420527, "lr": 0.0002991045903438766, "optimizer_lrs": [0.019940306022925108, 0.0002991045903438766], "gradient_norm": 0.5886387825012207, "seconds": 0.24483706500006974, "nonpadding_tokens_per_second": 322871.8658262689, "ranks": [{"compute_seconds": 0.2409770890000118, "nonpadding_tokens": 6131, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23886752099997466, "nonpadding_tokens": 3189, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2414774279999392, "nonpadding_tokens": 13017, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24183548300004531, "nonpadding_tokens": 10129, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2425742990000117, "nonpadding_tokens": 14435, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23954245000004448, "nonpadding_tokens": 13702, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2385591199999908, "nonpadding_tokens": 10810, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23912831400002688, "nonpadding_tokens": 7638, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003371489999608457, "prefetch": {"data_wait_seconds": 0.020884241001070336, "consumed_batches": 2223, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2773, "tokens": 202712033, "loss_tokens": 37163, "loss": 2.3198115397034687, "lr": 0.0002991042263896954, "optimizer_lrs": [0.01994028175931303, 0.0002991042263896954], "gradient_norm": 0.44971439242362976, "seconds": 0.26925033099996654, "nonpadding_tokens_per_second": 138023.97145429923, "ranks": [{"compute_seconds": 0.2669253740000386, "nonpadding_tokens": 5659, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26636172999997143, "nonpadding_tokens": 2975, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.263520643999982, "nonpadding_tokens": 3313, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26456942300001174, "nonpadding_tokens": 8025, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2662891289999152, "nonpadding_tokens": 2924, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26520613000002413, "nonpadding_tokens": 3535, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2660404270000072, "nonpadding_tokens": 5671, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2634264229999417, "nonpadding_tokens": 5061, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003344340000239754, "prefetch": {"data_wait_seconds": 0.02089195300102631, "consumed_batches": 2224, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2774, "tokens": 202799152, "loss_tokens": 87119, "loss": 2.6783838772254045, "lr": 0.000299103372904741, "optimizer_lrs": [0.019940224860316068, 0.000299103372904741], "gradient_norm": 0.6477750539779663, "seconds": 0.2431583730000284, "nonpadding_tokens_per_second": 358280.89703491237, "ranks": [{"compute_seconds": 0.23857001099997888, "nonpadding_tokens": 6131, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2407545470000514, "nonpadding_tokens": 13382, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23758178999992197, "nonpadding_tokens": 3275, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24011094499996943, "nonpadding_tokens": 8919, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24085101499997563, "nonpadding_tokens": 6307, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23923208899998372, "nonpadding_tokens": 6487, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23634367800002565, "nonpadding_tokens": 23630, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23709375399994315, "nonpadding_tokens": 18988, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003952209999624756, "prefetch": {"data_wait_seconds": 0.020901792000998398, "consumed_batches": 2225, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2775, "tokens": 202892164, "loss_tokens": 93012, "loss": 2.556428874768847, "lr": 0.00029910246124033833, "optimizer_lrs": [0.019940164082689223, 0.00029910246124033833], "gradient_norm": 0.9416922330856323, "seconds": 0.296717158999968, "nonpadding_tokens_per_second": 313470.2432224691, "ranks": [{"compute_seconds": 0.2916992749999281, "nonpadding_tokens": 19026, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2935686150000265, "nonpadding_tokens": 12734, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29045122799993806, "nonpadding_tokens": 10484, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2917359719999695, "nonpadding_tokens": 14669, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2943688759999077, "nonpadding_tokens": 18663, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29237583899998754, "nonpadding_tokens": 6087, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28952590100004727, "nonpadding_tokens": 6280, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29413289700005407, "nonpadding_tokens": 5069, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034929600008126727, "prefetch": {"data_wait_seconds": 0.02091142300101012, "consumed_batches": 2226, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2776, "tokens": 202946829, "loss_tokens": 54665, "loss": 2.301775873502241, "lr": 0.00029910192522164715, "optimizer_lrs": [0.01994012834810981, 0.00029910192522164715], "gradient_norm": 0.4150469899177551, "seconds": 0.33143207199998415, "nonpadding_tokens_per_second": 164935.75793715767, "ranks": [{"compute_seconds": 0.29737546500007284, "nonpadding_tokens": 10704, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30093523299990466, "nonpadding_tokens": 8683, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3003720689999909, "nonpadding_tokens": 5649, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3039374149999503, "nonpadding_tokens": 4327, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.30453253700000005, "nonpadding_tokens": 5219, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3020208709999679, "nonpadding_tokens": 9302, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29939899900000455, "nonpadding_tokens": 1260, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29963275400007205, "nonpadding_tokens": 9521, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.012302932000011424, "prefetch": {"data_wait_seconds": 0.02092059500098742, "consumed_batches": 2227, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2777, "tokens": 203015075, "loss_tokens": 68246, "loss": 2.1272586030683116, "lr": 0.000299101255810376, "optimizer_lrs": [0.019940083720691733, 0.000299101255810376], "gradient_norm": 0.4640754461288452, "seconds": 0.28156386400007705, "nonpadding_tokens_per_second": 242381.9556616872, "ranks": [{"compute_seconds": 0.27596451900001284, "nonpadding_tokens": 3260, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27552192999996805, "nonpadding_tokens": 11350, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27812454600007186, "nonpadding_tokens": 9638, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2753781620000382, "nonpadding_tokens": 6581, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27918084599991744, "nonpadding_tokens": 16816, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2766756839999971, "nonpadding_tokens": 12451, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2736100689999148, "nonpadding_tokens": 5996, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27921014199989713, "nonpadding_tokens": 2154, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038855699995110626, "prefetch": {"data_wait_seconds": 0.02092955500104665, "consumed_batches": 2228, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2778, "tokens": 203074106, "loss_tokens": 59031, "loss": 1.9755789066338025, "lr": 0.0002991006765867028, "optimizer_lrs": [0.019940045105780187, 0.0002991006765867028], "gradient_norm": 0.4861912429332733, "seconds": 0.28416608899999574, "nonpadding_tokens_per_second": 207734.14663141203, "ranks": [{"compute_seconds": 0.2809596550000606, "nonpadding_tokens": 2464, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28182840700003453, "nonpadding_tokens": 5620, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2804569290000245, "nonpadding_tokens": 4278, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27950663100000384, "nonpadding_tokens": 5220, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2775526330000275, "nonpadding_tokens": 4991, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2788288579999971, "nonpadding_tokens": 12842, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27745765999998184, "nonpadding_tokens": 13273, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2781032460000006, "nonpadding_tokens": 10343, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003842419999955382, "prefetch": {"data_wait_seconds": 0.02093939400101874, "consumed_batches": 2229, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2779, "tokens": 203113084, "loss_tokens": 38978, "loss": 2.3504244381445942, "lr": 0.00029910029402503653, "optimizer_lrs": [0.019940019601669103, 0.00029910029402503653], "gradient_norm": 0.45156341791152954, "seconds": 0.24963714399996206, "nonpadding_tokens_per_second": 156138.62334527398, "ranks": [{"compute_seconds": 0.24623372700000346, "nonpadding_tokens": 6609, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2446502520000422, "nonpadding_tokens": 3738, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24189923099993393, "nonpadding_tokens": 1681, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2460048650000317, "nonpadding_tokens": 2776, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24077145100000052, "nonpadding_tokens": 5935, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2438463269999147, "nonpadding_tokens": 2043, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24068670799999836, "nonpadding_tokens": 9267, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24129771699995217, "nonpadding_tokens": 6929, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003662689999828217, "prefetch": {"data_wait_seconds": 0.02094847800101718, "consumed_batches": 2230, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2780, "tokens": 203181359, "loss_tokens": 68275, "loss": 2.344010435737825, "lr": 0.00029909962372348594, "optimizer_lrs": [0.019939974914899065, 0.00029909962372348594], "gradient_norm": 0.44532641768455505, "seconds": 0.26806047299999136, "nonpadding_tokens_per_second": 254699.99077410492, "ranks": [{"compute_seconds": 0.20965239499992094, "nonpadding_tokens": 10026, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22737756399999398, "nonpadding_tokens": 5701, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.226552278999975, "nonpadding_tokens": 9879, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22524253799997496, "nonpadding_tokens": 18269, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2237890060000609, "nonpadding_tokens": 4239, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22454517399989982, "nonpadding_tokens": 8106, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2272526529999368, "nonpadding_tokens": 909, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22435059999997975, "nonpadding_tokens": 11146, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.02929423799992037, "prefetch": {"data_wait_seconds": 0.020956678000928264, "consumed_batches": 2231, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2781, "tokens": 203260115, "loss_tokens": 78756, "loss": 2.3243175519960384, "lr": 0.0002990988502139354, "optimizer_lrs": [0.019939923347595697, 0.0002990988502139354], "gradient_norm": 0.501625120639801, "seconds": 0.3036400290000074, "nonpadding_tokens_per_second": 259372.91686926456, "ranks": [{"compute_seconds": 0.30052740700000413, "nonpadding_tokens": 14460, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29760045399996216, "nonpadding_tokens": 12464, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2976735650000819, "nonpadding_tokens": 4274, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3003459000000248, "nonpadding_tokens": 20176, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2966116329999977, "nonpadding_tokens": 11669, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2990574269999797, "nonpadding_tokens": 5117, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30036161099997116, "nonpadding_tokens": 5557, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3013183639999397, "nonpadding_tokens": 5039, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003868300000249292, "prefetch": {"data_wait_seconds": 0.02096539700092137, "consumed_batches": 2232, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2782, "tokens": 203322640, "loss_tokens": 62525, "loss": 2.173422630947621, "lr": 0.0002990982358829887, "optimizer_lrs": [0.019939882392199248, 0.0002990982358829887], "gradient_norm": 0.4220310151576996, "seconds": 0.3907137490000423, "nonpadding_tokens_per_second": 160027.64212936166, "ranks": [{"compute_seconds": 0.3804350279999653, "nonpadding_tokens": 3160, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3787750720000531, "nonpadding_tokens": 12954, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3763056529999176, "nonpadding_tokens": 11947, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3769862579999881, "nonpadding_tokens": 11302, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3756947990000299, "nonpadding_tokens": 5232, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3779929719999018, "nonpadding_tokens": 3101, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37897572299993953, "nonpadding_tokens": 7328, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3791759959999581, "nonpadding_tokens": 7501, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034842200000184675, "prefetch": {"data_wait_seconds": 0.020974252000954152, "consumed_batches": 2233, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2783, "tokens": 203377613, "loss_tokens": 54973, "loss": 2.603646335473778, "lr": 0.00029909769558084733, "optimizer_lrs": [0.01993984637205649, 0.00029909769558084733], "gradient_norm": 0.4673503041267395, "seconds": 0.2944422809999878, "nonpadding_tokens_per_second": 186702.12651967016, "ranks": [{"compute_seconds": 0.27634237200004463, "nonpadding_tokens": 4944, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2790074450000475, "nonpadding_tokens": 4026, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2797498979999773, "nonpadding_tokens": 8836, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2799932869999111, "nonpadding_tokens": 3417, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2748169860000189, "nonpadding_tokens": 7116, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27700841399996534, "nonpadding_tokens": 15533, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2743018689999417, "nonpadding_tokens": 3440, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27523366800005533, "nonpadding_tokens": 7661, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005157030000191298, "prefetch": {"data_wait_seconds": 0.020985428000997786, "consumed_batches": 2234, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2784, "tokens": 203434832, "loss_tokens": 57219, "loss": 2.391797305090966, "lr": 0.00029909713303260356, "optimizer_lrs": [0.01993980886884024, 0.00029909713303260356], "gradient_norm": 0.4789637625217438, "seconds": 0.2383859469999834, "nonpadding_tokens_per_second": 240026.73278389178, "ranks": [{"compute_seconds": 0.23547597100002804, "nonpadding_tokens": 2776, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23608128499995473, "nonpadding_tokens": 5402, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23275271900001826, "nonpadding_tokens": 10739, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23377766900000552, "nonpadding_tokens": 7072, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.23592572099994413, "nonpadding_tokens": 7655, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23475764699992396, "nonpadding_tokens": 5025, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23199796300002617, "nonpadding_tokens": 15480, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2323936179999464, "nonpadding_tokens": 3070, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041925099992567993, "prefetch": {"data_wait_seconds": 0.020995365001070354, "consumed_batches": 2235, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2785, "tokens": 203486513, "loss_tokens": 51681, "loss": 2.392803478309243, "lr": 0.0002990966247810003, "optimizer_lrs": [0.019939774985400022, 0.0002990966247810003], "gradient_norm": 0.41222280263900757, "seconds": 0.29736521500001345, "nonpadding_tokens_per_second": 173796.38704546416, "ranks": [{"compute_seconds": 0.2941427240000394, "nonpadding_tokens": 5564, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.291448950999893, "nonpadding_tokens": 9610, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29414886500001103, "nonpadding_tokens": 3900, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29217815699996663, "nonpadding_tokens": 3745, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2950110589999895, "nonpadding_tokens": 6823, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2938281699999834, "nonpadding_tokens": 6289, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2946286420000206, "nonpadding_tokens": 7581, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29173374199990576, "nonpadding_tokens": 8169, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036765000004379544, "prefetch": {"data_wait_seconds": 0.0210029180011162, "consumed_batches": 2236, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2786, "tokens": 203573074, "loss_tokens": 86561, "loss": 2.2126701112510254, "lr": 0.0002990957731862466, "optimizer_lrs": [0.019939718212416444, 0.0002990957731862466], "gradient_norm": 0.38482940196990967, "seconds": 0.22194346300000234, "nonpadding_tokens_per_second": 390013.739670265, "ranks": [{"compute_seconds": 0.21889585799999622, "nonpadding_tokens": 2958, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21959317800008193, "nonpadding_tokens": 11997, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21636881499989613, "nonpadding_tokens": 6239, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21704274800003986, "nonpadding_tokens": 20126, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.21540432700010115, "nonpadding_tokens": 3570, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21817280400000527, "nonpadding_tokens": 6572, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21532571600005213, "nonpadding_tokens": 20259, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21561980499996025, "nonpadding_tokens": 14840, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041758400004709983, "prefetch": {"data_wait_seconds": 0.02101121700111719, "consumed_batches": 2237, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2787, "tokens": 203649582, "loss_tokens": 76508, "loss": 2.131626757986093, "lr": 0.0002990950201609008, "optimizer_lrs": [0.019939668010726722, 0.0002990950201609008], "gradient_norm": 0.4015903174877167, "seconds": 0.2639036409999562, "nonpadding_tokens_per_second": 289908.84593370464, "ranks": [{"compute_seconds": 0.2616512170000078, "nonpadding_tokens": 4040, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25781035600005, "nonpadding_tokens": 10901, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25758958900007656, "nonpadding_tokens": 13834, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.257582561999925, "nonpadding_tokens": 5085, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26154827400000613, "nonpadding_tokens": 18868, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2592900040000359, "nonpadding_tokens": 5582, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2564521079999622, "nonpadding_tokens": 2134, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2568476459999829, "nonpadding_tokens": 16064, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003425819999165469, "prefetch": {"data_wait_seconds": 0.021019927001091787, "consumed_batches": 2238, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2788, "tokens": 203710720, "loss_tokens": 61138, "loss": 1.9236895425103864, "lr": 0.00029909441818930444, "optimizer_lrs": [0.019939627879286964, 0.00029909441818930444], "gradient_norm": 0.38084152340888977, "seconds": 0.32442465700000866, "nonpadding_tokens_per_second": 188450.53444873757, "ranks": [{"compute_seconds": 0.3209377590000031, "nonpadding_tokens": 11899, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3220425479998994, "nonpadding_tokens": 6549, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3209769469999628, "nonpadding_tokens": 6745, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31928622500004167, "nonpadding_tokens": 8339, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.32174012899997706, "nonpadding_tokens": 1829, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3208231069999101, "nonpadding_tokens": 9315, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31785354999999527, "nonpadding_tokens": 3921, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31810662299994874, "nonpadding_tokens": 12541, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003426759999456408, "prefetch": {"data_wait_seconds": 0.02102875100099766, "consumed_batches": 2239, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2789, "tokens": 203797421, "loss_tokens": 86701, "loss": 2.0820672483593037, "lr": 0.00029909356417953995, "optimizer_lrs": [0.019939570945302666, 0.00029909356417953995], "gradient_norm": 0.39614617824554443, "seconds": 0.3463434820000657, "nonpadding_tokens_per_second": 250332.41422451125, "ranks": [{"compute_seconds": 0.3413466309999649, "nonpadding_tokens": 16838, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.339861768999981, "nonpadding_tokens": 11945, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34080179399995814, "nonpadding_tokens": 23749, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34301543900005527, "nonpadding_tokens": 5532, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3394397460000391, "nonpadding_tokens": 4285, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3411457329999621, "nonpadding_tokens": 8205, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3434522619999143, "nonpadding_tokens": 6431, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34395113599998695, "nonpadding_tokens": 9716, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003782320000027539, "prefetch": {"data_wait_seconds": 0.02103889000098036, "consumed_batches": 2240, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2790, "tokens": 203828314, "loss_tokens": 30893, "loss": 2.522985081571877, "lr": 0.0002990932597848074, "optimizer_lrs": [0.019939550652320493, 0.0002990932597848074], "gradient_norm": 0.48780834674835205, "seconds": 0.19054351600004793, "nonpadding_tokens_per_second": 162130.94335885052, "ranks": [{"compute_seconds": 0.18835552400003053, "nonpadding_tokens": 3623, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.1877957009999136, "nonpadding_tokens": 5454, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.1848427579999452, "nonpadding_tokens": 2197, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.18598661199996513, "nonpadding_tokens": 6250, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.18714760800003205, "nonpadding_tokens": 5193, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.18639031400005024, "nonpadding_tokens": 1905, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.18714996899996095, "nonpadding_tokens": 3248, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.18786879100002807, "nonpadding_tokens": 3023, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003958089999969161, "prefetch": {"data_wait_seconds": 0.021047484001044836, "consumed_batches": 2241, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2791, "tokens": 203878872, "loss_tokens": 50558, "loss": 2.214897524872424, "lr": 0.00029909276151713184, "optimizer_lrs": [0.019939517434475456, 0.00029909276151713184], "gradient_norm": 0.4359332323074341, "seconds": 0.3225899120000122, "nonpadding_tokens_per_second": 156725.29772102137, "ranks": [{"compute_seconds": 0.3199638449999611, "nonpadding_tokens": 5700, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3202929410000479, "nonpadding_tokens": 10872, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31681260599998495, "nonpadding_tokens": 2994, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3198074680000218, "nonpadding_tokens": 9527, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3164267249999284, "nonpadding_tokens": 2501, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31749436600000536, "nonpadding_tokens": 4708, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3160814400000618, "nonpadding_tokens": 2906, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31682761800004755, "nonpadding_tokens": 11350, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037184099994647113, "prefetch": {"data_wait_seconds": 0.021055734000995585, "consumed_batches": 2242, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2792, "tokens": 203956284, "loss_tokens": 77412, "loss": 1.970713923551904, "lr": 0.00029909199832909303, "optimizer_lrs": [0.01993946655527287, 0.00029909199832909303], "gradient_norm": 0.5796966552734375, "seconds": 0.3688524750000397, "nonpadding_tokens_per_second": 209872.52423883468, "ranks": [{"compute_seconds": 0.3642271859999937, "nonpadding_tokens": 18293, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36654318399996555, "nonpadding_tokens": 19327, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3635685149999972, "nonpadding_tokens": 5936, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3635473460000185, "nonpadding_tokens": 7748, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3628480950000039, "nonpadding_tokens": 5921, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36508376699998735, "nonpadding_tokens": 5808, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3662510830000656, "nonpadding_tokens": 11684, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3630622300000823, "nonpadding_tokens": 2695, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039216599998326274, "prefetch": {"data_wait_seconds": 0.021065998001020603, "consumed_batches": 2243, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2793, "tokens": 204009315, "loss_tokens": 53031, "loss": 2.0671604521412004, "lr": 0.0002990914753234487, "optimizer_lrs": [0.019939431688229915, 0.0002990914753234487], "gradient_norm": 0.4079783260822296, "seconds": 0.33921489399995153, "nonpadding_tokens_per_second": 156334.52698573895, "ranks": [{"compute_seconds": 0.33651535600006355, "nonpadding_tokens": 9521, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33441555299998527, "nonpadding_tokens": 5478, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33685837699999865, "nonpadding_tokens": 1642, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33533602900001824, "nonpadding_tokens": 11518, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3338709030000473, "nonpadding_tokens": 8725, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3366318419999743, "nonpadding_tokens": 10605, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33393422500000725, "nonpadding_tokens": 2399, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33462003300007837, "nonpadding_tokens": 3143, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039849600000252394, "prefetch": {"data_wait_seconds": 0.021074080001042717, "consumed_batches": 2244, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2794, "tokens": 204066796, "loss_tokens": 57481, "loss": 2.395528250639342, "lr": 0.00029909090826124536, "optimizer_lrs": [0.019939393884083025, 0.00029909090826124536], "gradient_norm": 0.42430731654167175, "seconds": 0.2710942690000593, "nonpadding_tokens_per_second": 212033.25401167898, "ranks": [{"compute_seconds": 0.26824863200010896, "nonpadding_tokens": 2250, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2652562260000195, "nonpadding_tokens": 12439, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26577697300001546, "nonpadding_tokens": 9282, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2661334470000156, "nonpadding_tokens": 3024, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.268332436000037, "nonpadding_tokens": 10064, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2672950830000218, "nonpadding_tokens": 10703, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2681662149999511, "nonpadding_tokens": 4113, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26529268499996306, "nonpadding_tokens": 5606, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003467559999990044, "prefetch": {"data_wait_seconds": 0.021081868000990767, "consumed_batches": 2245, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2795, "tokens": 204120027, "loss_tokens": 53231, "loss": 2.545980314572336, "lr": 0.00029909038296894704, "optimizer_lrs": [0.01993935886459647, 0.00029909038296894704], "gradient_norm": 0.4073217213153839, "seconds": 0.26757720299997345, "nonpadding_tokens_per_second": 198936.97745246737, "ranks": [{"compute_seconds": 0.2451768160000256, "nonpadding_tokens": 3637, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24625686299998506, "nonpadding_tokens": 8962, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24762238100004197, "nonpadding_tokens": 11628, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24836002900008225, "nonpadding_tokens": 3717, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24633216099994115, "nonpadding_tokens": 6793, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2470668860000842, "nonpadding_tokens": 3681, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25015175100008946, "nonpadding_tokens": 3315, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24654800899998008, "nonpadding_tokens": 11498, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0008238010000241047, "prefetch": {"data_wait_seconds": 0.02109145300103421, "consumed_batches": 2246, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2796, "tokens": 204208198, "loss_tokens": 88171, "loss": 2.212781838699799, "lr": 0.0002990895125503073, "optimizer_lrs": [0.019939300836687157, 0.0002990895125503073], "gradient_norm": 0.39870965480804443, "seconds": 0.23069505500006926, "nonpadding_tokens_per_second": 382197.18233654177, "ranks": [{"compute_seconds": 0.22676833199989233, "nonpadding_tokens": 8147, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22810805500000697, "nonpadding_tokens": 9385, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22726298400004907, "nonpadding_tokens": 20499, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22583120300009796, "nonpadding_tokens": 24077, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.22415390999992724, "nonpadding_tokens": 7615, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22556038599998374, "nonpadding_tokens": 2776, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22805467800003498, "nonpadding_tokens": 6378, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.224387997000008, "nonpadding_tokens": 9294, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036319699995601695, "prefetch": {"data_wait_seconds": 0.021100816000924283, "consumed_batches": 2247, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2797, "tokens": 204267874, "loss_tokens": 59676, "loss": 2.5684579542026946, "lr": 0.0002990889231971326, "optimizer_lrs": [0.019939261546475507, 0.0002990889231971326], "gradient_norm": 0.42895036935806274, "seconds": 0.320789017999914, "nonpadding_tokens_per_second": 186028.81224573593, "ranks": [{"compute_seconds": 0.2822803679999879, "nonpadding_tokens": 4260, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29847072000006847, "nonpadding_tokens": 5602, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2980586170000379, "nonpadding_tokens": 11934, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2963566720000017, "nonpadding_tokens": 3595, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2989001889999372, "nonpadding_tokens": 9964, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2971633299999894, "nonpadding_tokens": 9435, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29846413899997515, "nonpadding_tokens": 10737, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29497161799997684, "nonpadding_tokens": 4149, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.013804986000081954, "prefetch": {"data_wait_seconds": 0.02110931200088828, "consumed_batches": 2248, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2798, "tokens": 204322550, "loss_tokens": 54676, "loss": 2.1298735962762456, "lr": 0.00029908838305655266, "optimizer_lrs": [0.019939225537103512, 0.00029908838305655266], "gradient_norm": 0.46011391282081604, "seconds": 0.24812693600006241, "nonpadding_tokens_per_second": 220354.95573921184, "ranks": [{"compute_seconds": 0.24332018899997365, "nonpadding_tokens": 3696, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2452343339999743, "nonpadding_tokens": 4332, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2424455839999382, "nonpadding_tokens": 9248, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24286059899998236, "nonpadding_tokens": 7122, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.24089911500004746, "nonpadding_tokens": 12281, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24373536899997816, "nonpadding_tokens": 9967, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2414070410000022, "nonpadding_tokens": 2267, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24575157399999625, "nonpadding_tokens": 5763, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003513210000392064, "prefetch": {"data_wait_seconds": 0.021119701000955615, "consumed_batches": 2249, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2799, "tokens": 204389870, "loss_tokens": 67320, "loss": 2.31947090389186, "lr": 0.00029908771778760986, "optimizer_lrs": [0.01993918118584066, 0.00029908771778760986], "gradient_norm": 0.4207073152065277, "seconds": 0.30250876800005244, "nonpadding_tokens_per_second": 222539.00422479102, "ranks": [{"compute_seconds": 0.29819352600009097, "nonpadding_tokens": 9867, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3002002519999678, "nonpadding_tokens": 5308, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.297228763000021, "nonpadding_tokens": 9786, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30013095300000714, "nonpadding_tokens": 5876, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.29609446599999956, "nonpadding_tokens": 5659, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2988022129999308, "nonpadding_tokens": 10052, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30016513099997155, "nonpadding_tokens": 12121, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29692147399998703, "nonpadding_tokens": 8651, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003767880000395962, "prefetch": {"data_wait_seconds": 0.021128927000930275, "consumed_batches": 2250, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2800, "tokens": 204437197, "loss_tokens": 47327, "loss": 2.3997940855642654, "lr": 0.00029908724994849784, "optimizer_lrs": [0.019939149996566524, 0.00029908724994849784], "gradient_norm": 0.40720534324645996, "seconds": 0.21655089500006852, "nonpadding_tokens_per_second": 218549.08519304445, "ranks": [{"compute_seconds": 0.2136530299999322, "nonpadding_tokens": 3517, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21090808999997535, "nonpadding_tokens": 6112, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2113109809999969, "nonpadding_tokens": 7790, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21421998700009226, "nonpadding_tokens": 7259, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.2102487710000105, "nonpadding_tokens": 5209, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21318053300001338, "nonpadding_tokens": 4885, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20996622699999534, "nonpadding_tokens": 9457, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2111850679999634, "nonpadding_tokens": 3098, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003402720000167392, "prefetch": {"data_wait_seconds": 0.02113752000089164, "consumed_batches": 2251, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2801, "tokens": 204554981, "loss_tokens": 117784, "loss": 2.106894559108198, "lr": 0.0002990860851056254, "optimizer_lrs": [0.01993907234037503, 0.0002990860851056254], "gradient_norm": 0.4844464659690857, "seconds": 0.3264037479999615, "nonpadding_tokens_per_second": 360853.69951087044, "ranks": [{"compute_seconds": 0.32149835699999585, "nonpadding_tokens": 1398, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32404912700008026, "nonpadding_tokens": 9450, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32118980500001726, "nonpadding_tokens": 18965, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3232147899999518, "nonpadding_tokens": 14342, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.31947230000002946, "nonpadding_tokens": 25520, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.321696291999956, "nonpadding_tokens": 28263, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31957006300001467, "nonpadding_tokens": 16352, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3201732180000363, "nonpadding_tokens": 3494, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003983340000104363, "prefetch": {"data_wait_seconds": 0.021146921000948282, "consumed_batches": 2252, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2802, "tokens": 204603319, "loss_tokens": 48338, "loss": 2.601279984173942, "lr": 0.00029908560684534024, "optimizer_lrs": [0.019939040456356017, 0.00029908560684534024], "gradient_norm": 0.46744272112846375, "seconds": 0.2702837790000103, "nonpadding_tokens_per_second": 178841.66108243645, "ranks": [{"compute_seconds": 0.26666165500000716, "nonpadding_tokens": 6445, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2649584370000184, "nonpadding_tokens": 9126, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2646939569999631, "nonpadding_tokens": 4952, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2680352220000941, "nonpadding_tokens": 3054, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.26401230499993744, "nonpadding_tokens": 7778, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2667280649999384, "nonpadding_tokens": 8467, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2634764350000296, "nonpadding_tokens": 2984, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2644871920000469, "nonpadding_tokens": 5532, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003420330000381, "prefetch": {"data_wait_seconds": 0.021155590001015412, "consumed_batches": 2253, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2803, "tokens": 204655438, "loss_tokens": 52119, "loss": 2.0700749126997833, "lr": 0.0002990850910358279, "optimizer_lrs": [0.019939006069055194, 0.0002990850910358279], "gradient_norm": 0.9169502854347229, "seconds": 0.3568281510000588, "nonpadding_tokens_per_second": 146061.9064217033, "ranks": [{"compute_seconds": 0.3531305679999832, "nonpadding_tokens": 8114, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35453384599998117, "nonpadding_tokens": 10907, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35128504800002247, "nonpadding_tokens": 3560, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3537604289999763, "nonpadding_tokens": 6631, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.35083152300001075, "nonpadding_tokens": 7306, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35192977500003053, "nonpadding_tokens": 7033, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3503380229999493, "nonpadding_tokens": 3865, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35098598799993397, "nonpadding_tokens": 4703, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003611769999452008, "prefetch": {"data_wait_seconds": 0.02116509400100597, "consumed_batches": 2254, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2804, "tokens": 204706266, "loss_tokens": 50828, "loss": 2.2385434701345717, "lr": 0.0002990845878634323, "optimizer_lrs": [0.01993897252422882, 0.0002990845878634323], "gradient_norm": 0.4801064431667328, "seconds": 0.30608904099995016, "nonpadding_tokens_per_second": 166056.25550641087, "ranks": [{"compute_seconds": 0.30018313400000807, "nonpadding_tokens": 2710, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3030815449999409, "nonpadding_tokens": 7610, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3000072749999845, "nonpadding_tokens": 7416, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30310002900000654, "nonpadding_tokens": 1656, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.3037273829999094, "nonpadding_tokens": 11759, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3017774779999627, "nonpadding_tokens": 2495, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2989256399999931, "nonpadding_tokens": 5984, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2994490300000052, "nonpadding_tokens": 11198, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040110599991294293, "prefetch": {"data_wait_seconds": 0.021175223000980736, "consumed_batches": 2255, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2805, "tokens": 204778385, "loss_tokens": 72119, "loss": 2.1610159943981473, "lr": 0.0002990838736839581, "optimizer_lrs": [0.019938924912263872, 0.0002990838736839581], "gradient_norm": 0.38431572914123535, "seconds": 0.27921457600007216, "nonpadding_tokens_per_second": 258292.38943450202, "ranks": [{"compute_seconds": 0.27590598099993713, "nonpadding_tokens": 12207, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2763882259999946, "nonpadding_tokens": 9145, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27323940899998433, "nonpadding_tokens": 5353, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2760225269998955, "nonpadding_tokens": 18665, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17869832192}, {"compute_seconds": 0.27196959699995205, "nonpadding_tokens": 18272, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.275046009000107, "nonpadding_tokens": 3040, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2720443709999927, "nonpadding_tokens": 1780, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.276857494000069, "nonpadding_tokens": 3657, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034549000008610165, "prefetch": {"data_wait_seconds": 0.021183915001032005, "consumed_batches": 2256, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2806, "tokens": 204864210, "loss_tokens": 85825, "loss": 2.5657373288668803, "lr": 0.00029908302341515004, "optimizer_lrs": [0.01993886822767667, 0.00029908302341515004], "gradient_norm": 0.3843872547149658, "seconds": 0.31164343099999314, "nonpadding_tokens_per_second": 275394.8630478333, "ranks": [{"compute_seconds": 0.26509541799998715, "nonpadding_tokens": 18521, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2685034569999516, "nonpadding_tokens": 10472, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2684494160000668, "nonpadding_tokens": 12730, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26921485299999404, "nonpadding_tokens": 9851, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27191879600002267, "nonpadding_tokens": 5196, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.270325431999936, "nonpadding_tokens": 12760, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2715782110000191, "nonpadding_tokens": 5602, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26819624200004455, "nonpadding_tokens": 10693, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.013834620000011455, "prefetch": {"data_wait_seconds": 0.021194615000993622, "consumed_batches": 2257, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2807, "tokens": 204977379, "loss_tokens": 113169, "loss": 2.2262136715885092, "lr": 0.0002990819016483336, "optimizer_lrs": [0.019938793443222243, 0.0002990819016483336], "gradient_norm": 0.4245541989803314, "seconds": 0.22890862100007325, "nonpadding_tokens_per_second": 494385.0498315823, "ranks": [{"compute_seconds": 0.22591993399998955, "nonpadding_tokens": 5179, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22504363099994862, "nonpadding_tokens": 17346, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22506796199991186, "nonpadding_tokens": 5870, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2259665760000189, "nonpadding_tokens": 28087, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22374436999996306, "nonpadding_tokens": 18019, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22673670599999696, "nonpadding_tokens": 17197, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.224142780999955, "nonpadding_tokens": 4328, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22470229800001107, "nonpadding_tokens": 17143, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034219100007248926, "prefetch": {"data_wait_seconds": 0.021203156000936474, "consumed_batches": 2258, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2808, "tokens": 205084725, "loss_tokens": 107346, "loss": 2.3814518298306413, "lr": 0.00029908083696938303, "optimizer_lrs": [0.01993872246462554, 0.00029908083696938303], "gradient_norm": 0.391687273979187, "seconds": 0.3073289980000027, "nonpadding_tokens_per_second": 349286.9228044633, "ranks": [{"compute_seconds": 0.3051569379999819, "nonpadding_tokens": 27815, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3046850609999865, "nonpadding_tokens": 3115, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30467483099994297, "nonpadding_tokens": 10100, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3023498469999595, "nonpadding_tokens": 22089, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3004808690000118, "nonpadding_tokens": 14340, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30361043400000653, "nonpadding_tokens": 15769, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3002393330000359, "nonpadding_tokens": 10015, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3008514919999925, "nonpadding_tokens": 4103, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003291279999757535, "prefetch": {"data_wait_seconds": 0.021211314000950665, "consumed_batches": 2259, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2809, "tokens": 205131662, "loss_tokens": 46937, "loss": 2.0338047875343546, "lr": 0.00029908037124569165, "optimizer_lrs": [0.019938691416379446, 0.00029908037124569165], "gradient_norm": 0.4137480854988098, "seconds": 0.22628403400005936, "nonpadding_tokens_per_second": 207425.15134756564, "ranks": [{"compute_seconds": 0.22201377700002922, "nonpadding_tokens": 3039, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.224003714000105, "nonpadding_tokens": 9619, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22279695099996388, "nonpadding_tokens": 7673, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22351956599993628, "nonpadding_tokens": 945, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21988877599994794, "nonpadding_tokens": 6711, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2223603820000335, "nonpadding_tokens": 11811, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21930865999991056, "nonpadding_tokens": 4685, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22016105399995922, "nonpadding_tokens": 2454, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037151900005483185, "prefetch": {"data_wait_seconds": 0.021219978000956985, "consumed_batches": 2260, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2810, "tokens": 205192870, "loss_tokens": 61208, "loss": 2.184338648542674, "lr": 0.0002990797637440453, "optimizer_lrs": [0.019938650916269687, 0.0002990797637440453], "gradient_norm": 0.41651231050491333, "seconds": 0.3279330429999163, "nonpadding_tokens_per_second": 186647.85786778925, "ranks": [{"compute_seconds": 0.32492382400005226, "nonpadding_tokens": 15396, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3219063210000286, "nonpadding_tokens": 13016, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3223779889999605, "nonpadding_tokens": 1994, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3230078029999959, "nonpadding_tokens": 1681, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32560701599993536, "nonpadding_tokens": 6764, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3235114540000268, "nonpadding_tokens": 3580, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32105970800000705, "nonpadding_tokens": 2285, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3216062430000193, "nonpadding_tokens": 16492, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033559299993157765, "prefetch": {"data_wait_seconds": 0.021228337000891884, "consumed_batches": 2261, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2811, "tokens": 205256489, "loss_tokens": 63619, "loss": 2.3764834404816173, "lr": 0.00029907913210087137, "optimizer_lrs": [0.01993860880672476, 0.00029907913210087137], "gradient_norm": 0.3964080810546875, "seconds": 0.2751730029999635, "nonpadding_tokens_per_second": 231196.37212378875, "ranks": [{"compute_seconds": 0.27094416600004934, "nonpadding_tokens": 14408, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2727770239999927, "nonpadding_tokens": 2728, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2703297680000105, "nonpadding_tokens": 5578, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2710277759999826, "nonpadding_tokens": 7854, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26957298900003934, "nonpadding_tokens": 4714, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27114556200001516, "nonpadding_tokens": 13281, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2690958430000592, "nonpadding_tokens": 10967, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2695744540000078, "nonpadding_tokens": 4089, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038787500000125874, "prefetch": {"data_wait_seconds": 0.021239412000866764, "consumed_batches": 2262, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2812, "tokens": 205310192, "loss_tokens": 53703, "loss": 2.4645268304377783, "lr": 0.00029907859874091144, "optimizer_lrs": [0.019938573249394097, 0.00029907859874091144], "gradient_norm": 0.43114718794822693, "seconds": 0.33558067600006325, "nonpadding_tokens_per_second": 160030.07276852222, "ranks": [{"compute_seconds": 0.32968501800007743, "nonpadding_tokens": 6648, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3323538769999459, "nonpadding_tokens": 4307, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32920414499994877, "nonpadding_tokens": 5134, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32964235700001154, "nonpadding_tokens": 7093, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3279102519999242, "nonpadding_tokens": 3532, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3308208889999378, "nonpadding_tokens": 10096, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33233346800000163, "nonpadding_tokens": 12002, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3331984269999566, "nonpadding_tokens": 4891, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034445099993263284, "prefetch": {"data_wait_seconds": 0.02124703900081215, "consumed_batches": 2263, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2813, "tokens": 205360599, "loss_tokens": 50407, "loss": 2.4075475628384946, "lr": 0.00029907809797569585, "optimizer_lrs": [0.01993853986504639, 0.00029907809797569585], "gradient_norm": 0.5494180917739868, "seconds": 0.4365507019999768, "nonpadding_tokens_per_second": 115466.54207419572, "ranks": [{"compute_seconds": 0.4321617560000277, "nonpadding_tokens": 6257, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.43404079399999773, "nonpadding_tokens": 12386, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.43120553400001427, "nonpadding_tokens": 6520, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4316400139999814, "nonpadding_tokens": 2322, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.42978317699999025, "nonpadding_tokens": 870, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4330671310000298, "nonpadding_tokens": 9207, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.43416724700000486, "nonpadding_tokens": 11294, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.430435287000023, "nonpadding_tokens": 1551, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003341240000054313, "prefetch": {"data_wait_seconds": 0.02125538400078142, "consumed_batches": 2264, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2814, "tokens": 205427960, "loss_tokens": 67361, "loss": 2.330303792253678, "lr": 0.00029907742857037545, "optimizer_lrs": [0.01993849523802503, 0.00029907742857037545], "gradient_norm": 0.43011370301246643, "seconds": 0.22982786499994745, "nonpadding_tokens_per_second": 293093.2678681734, "ranks": [{"compute_seconds": 0.22553005400004622, "nonpadding_tokens": 10275, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2274024380000128, "nonpadding_tokens": 2619, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22418347800009997, "nonpadding_tokens": 8181, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22488098099995568, "nonpadding_tokens": 3481, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22746328099992752, "nonpadding_tokens": 11707, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22589885100001084, "nonpadding_tokens": 11899, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22343641300005856, "nonpadding_tokens": 11615, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22388348900005894, "nonpadding_tokens": 7584, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041631600004166103, "prefetch": {"data_wait_seconds": 0.021266732000825073, "consumed_batches": 2265, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2815, "tokens": 205467396, "loss_tokens": 39436, "loss": 1.8476602121158332, "lr": 0.0002990770365596122, "optimizer_lrs": [0.019938469103974146, 0.0002990770365596122], "gradient_norm": 0.4775609076023102, "seconds": 0.27288668200003485, "nonpadding_tokens_per_second": 144514.1980215618, "ranks": [{"compute_seconds": 0.26662085099997057, "nonpadding_tokens": 6321, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26869498500002464, "nonpadding_tokens": 2802, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2689200830000118, "nonpadding_tokens": 2636, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2659808130000556, "nonpadding_tokens": 2129, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2699026340000046, "nonpadding_tokens": 4745, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26762560099996335, "nonpadding_tokens": 4458, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2647409420000031, "nonpadding_tokens": 13842, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27042756999992434, "nonpadding_tokens": 2503, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033917700000074547, "prefetch": {"data_wait_seconds": 0.021274866000752013, "consumed_batches": 2266, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2816, "tokens": 205569103, "loss_tokens": 101707, "loss": 2.1039127714906547, "lr": 0.0002990760251654074, "optimizer_lrs": [0.01993840167769383, 0.0002990760251654074], "gradient_norm": 0.43252304196357727, "seconds": 0.25743056400006026, "nonpadding_tokens_per_second": 395085.17722074443, "ranks": [{"compute_seconds": 0.2547520220000479, "nonpadding_tokens": 9445, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2550338239999519, "nonpadding_tokens": 9751, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25226695100002416, "nonpadding_tokens": 11707, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25248837499998444, "nonpadding_tokens": 10852, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25141405800002303, "nonpadding_tokens": 23551, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25388795600008507, "nonpadding_tokens": 9872, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25124588299991046, "nonpadding_tokens": 10356, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25188683799990486, "nonpadding_tokens": 16173, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036414399994555424, "prefetch": {"data_wait_seconds": 0.021283953000832412, "consumed_batches": 2267, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2817, "tokens": 205632043, "loss_tokens": 62940, "loss": 2.658747914680648, "lr": 0.00029907539900132947, "optimizer_lrs": [0.019938359933421967, 0.00029907539900132947], "gradient_norm": 0.5276370048522949, "seconds": 0.3577037960000098, "nonpadding_tokens_per_second": 175955.6390058502, "ranks": [{"compute_seconds": 0.35455268199996226, "nonpadding_tokens": 10010, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35393969599999764, "nonpadding_tokens": 7886, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3538118509999322, "nonpadding_tokens": 15048, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3544440470000154, "nonpadding_tokens": 12202, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.35299557400003323, "nonpadding_tokens": 3321, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3554267910000135, "nonpadding_tokens": 7317, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35234982399992987, "nonpadding_tokens": 4174, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35328036199996404, "nonpadding_tokens": 2982, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036092600009851594, "prefetch": {"data_wait_seconds": 0.021292525000902174, "consumed_batches": 2268, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2818, "tokens": 205683937, "loss_tokens": 51894, "loss": 2.1482541334258296, "lr": 0.0002990748825703637, "optimizer_lrs": [0.019938325504690917, 0.0002990748825703637], "gradient_norm": 0.4344065189361572, "seconds": 0.26311980399998447, "nonpadding_tokens_per_second": 197225.7474013741, "ranks": [{"compute_seconds": 0.2596377400000165, "nonpadding_tokens": 4593, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26062787000000753, "nonpadding_tokens": 4410, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2582204999999931, "nonpadding_tokens": 2884, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2583402239999941, "nonpadding_tokens": 15825, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25691060899998774, "nonpadding_tokens": 2283, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25983219400006874, "nonpadding_tokens": 6968, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2565676430000394, "nonpadding_tokens": 4042, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2576785160000554, "nonpadding_tokens": 10889, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003482990000520658, "prefetch": {"data_wait_seconds": 0.021302955000990664, "consumed_batches": 2269, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2819, "tokens": 205745866, "loss_tokens": 61929, "loss": 2.504760996463692, "lr": 0.0002990742660865284, "optimizer_lrs": [0.01993828440576856, 0.0002990742660865284], "gradient_norm": 0.4540971517562866, "seconds": 0.29881615699991926, "nonpadding_tokens_per_second": 207247.82964134277, "ranks": [{"compute_seconds": 0.2959776120000015, "nonpadding_tokens": 5170, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29168347799998173, "nonpadding_tokens": 8187, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2900055849999035, "nonpadding_tokens": 5129, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.291025584999943, "nonpadding_tokens": 4816, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28874065000002247, "nonpadding_tokens": 20187, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2919041019999895, "nonpadding_tokens": 3663, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2888273969999773, "nonpadding_tokens": 2940, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2892774339999278, "nonpadding_tokens": 11837, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046236100001806335, "prefetch": {"data_wait_seconds": 0.02131421000099465, "consumed_batches": 2270, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2820, "tokens": 205797852, "loss_tokens": 51986, "loss": 2.874163236255915, "lr": 0.00029907374842416185, "optimizer_lrs": [0.019938249894944124, 0.00029907374842416185], "gradient_norm": 0.5896766781806946, "seconds": 0.2734492769999406, "nonpadding_tokens_per_second": 190112.04041329864, "ranks": [{"compute_seconds": 0.2684247959999766, "nonpadding_tokens": 5059, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2708566640000072, "nonpadding_tokens": 5044, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2673350609999261, "nonpadding_tokens": 12186, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27047530599998026, "nonpadding_tokens": 4135, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2709337610000375, "nonpadding_tokens": 4936, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26890240900002027, "nonpadding_tokens": 7948, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26687520300004053, "nonpadding_tokens": 4994, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.267283410999994, "nonpadding_tokens": 7684, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004570759999751317, "prefetch": {"data_wait_seconds": 0.021323784001040735, "consumed_batches": 2271, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2821, "tokens": 205864475, "loss_tokens": 66623, "loss": 2.5858531869624604, "lr": 0.00029907308479974087, "optimizer_lrs": [0.019938205653316058, 0.00029907308479974087], "gradient_norm": 0.41530030965805054, "seconds": 0.22731171800000993, "nonpadding_tokens_per_second": 293090.9175566439, "ranks": [{"compute_seconds": 0.22152651800001877, "nonpadding_tokens": 4666, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22391789699997844, "nonpadding_tokens": 7406, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22046414100009315, "nonpadding_tokens": 5385, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22174496800005272, "nonpadding_tokens": 18746, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22007106599994586, "nonpadding_tokens": 8712, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2218562350000184, "nonpadding_tokens": 7851, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2243296479999799, "nonpadding_tokens": 3556, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2250235789999806, "nonpadding_tokens": 10301, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033692299996346264, "prefetch": {"data_wait_seconds": 0.02133206800097298, "consumed_batches": 2272, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2822, "tokens": 205950128, "loss_tokens": 85653, "loss": 2.244616906296335, "lr": 0.00029907223127161216, "optimizer_lrs": [0.01993814875144081, 0.00029907223127161216], "gradient_norm": 0.5172687768936157, "seconds": 0.3352684640000234, "nonpadding_tokens_per_second": 255475.8624717952, "ranks": [{"compute_seconds": 0.32854727400001593, "nonpadding_tokens": 6619, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3312399359999745, "nonpadding_tokens": 18607, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3322843810000222, "nonpadding_tokens": 10776, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32847529100001793, "nonpadding_tokens": 16879, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3270145110000158, "nonpadding_tokens": 7972, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3298155270000507, "nonpadding_tokens": 3164, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3326578809999319, "nonpadding_tokens": 11143, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32753229399997963, "nonpadding_tokens": 10493, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003748120000182098, "prefetch": {"data_wait_seconds": 0.021339644001002966, "consumed_batches": 2273, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2823, "tokens": 206031732, "loss_tokens": 81604, "loss": 2.5106849848046666, "lr": 0.0002990714177274577, "optimizer_lrs": [0.01993809451516385, 0.0002990714177274577], "gradient_norm": 0.8531784415245056, "seconds": 0.2929399749999675, "nonpadding_tokens_per_second": 278569.014010495, "ranks": [{"compute_seconds": 0.2877303400000528, "nonpadding_tokens": 12150, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2900484230000302, "nonpadding_tokens": 2060, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2894413459999896, "nonpadding_tokens": 2890, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28739170399990144, "nonpadding_tokens": 5319, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2856287480000219, "nonpadding_tokens": 8366, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28889741099999355, "nonpadding_tokens": 21858, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28634965000003376, "nonpadding_tokens": 10675, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2905992100000958, "nonpadding_tokens": 18286, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003406409999797688, "prefetch": {"data_wait_seconds": 0.021348044001072708, "consumed_batches": 2274, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2824, "tokens": 206096481, "loss_tokens": 64749, "loss": 2.4986375272204975, "lr": 0.0002990707719649927, "optimizer_lrs": [0.019938051464332848, 0.0002990707719649927], "gradient_norm": 0.4117635488510132, "seconds": 0.2654886970000234, "nonpadding_tokens_per_second": 243886.08905634235, "ranks": [{"compute_seconds": 0.26173981400006596, "nonpadding_tokens": 11059, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2629457600000933, "nonpadding_tokens": 11686, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2596355959999528, "nonpadding_tokens": 4041, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26293076700005713, "nonpadding_tokens": 7902, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25833064200003264, "nonpadding_tokens": 9849, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2613054099999772, "nonpadding_tokens": 7525, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25900425600002563, "nonpadding_tokens": 5525, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.263151138000012, "nonpadding_tokens": 7162, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034958200001256046, "prefetch": {"data_wait_seconds": 0.021356289001076334, "consumed_batches": 2275, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2825, "tokens": 206140915, "loss_tokens": 44434, "loss": 2.414126147769726, "lr": 0.00029907032868108926, "optimizer_lrs": [0.01993802191207262, 0.00029907032868108926], "gradient_norm": 0.4538801610469818, "seconds": 0.25613388399995074, "nonpadding_tokens_per_second": 173479.58538749424, "ranks": [{"compute_seconds": 0.25242792999995345, "nonpadding_tokens": 3312, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25132934800001294, "nonpadding_tokens": 10955, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24814644200000657, "nonpadding_tokens": 3362, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24853121499995723, "nonpadding_tokens": 6704, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24705875400002242, "nonpadding_tokens": 4046, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2497969830000102, "nonpadding_tokens": 4147, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24696009299998423, "nonpadding_tokens": 7805, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25353797299999314, "nonpadding_tokens": 4103, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003441269999484575, "prefetch": {"data_wait_seconds": 0.02136469900104032, "consumed_batches": 2276, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2826, "tokens": 206201905, "loss_tokens": 60990, "loss": 2.387281982702082, "lr": 0.0002990697200591542, "optimizer_lrs": [0.019937981337276946, 0.0002990697200591542], "gradient_norm": 0.6401089429855347, "seconds": 0.2942946339999253, "nonpadding_tokens_per_second": 207241.29139240604, "ranks": [{"compute_seconds": 0.2896104420000256, "nonpadding_tokens": 5037, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2881896520000282, "nonpadding_tokens": 9780, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2885890710000467, "nonpadding_tokens": 4782, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28932685000006586, "nonpadding_tokens": 6040, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2877379189999374, "nonpadding_tokens": 11419, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2906127459999652, "nonpadding_tokens": 7843, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2919072339999502, "nonpadding_tokens": 12112, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2881031230000417, "nonpadding_tokens": 3977, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004408450000710218, "prefetch": {"data_wait_seconds": 0.021375448001052177, "consumed_batches": 2277, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2827, "tokens": 206268403, "loss_tokens": 66498, "loss": 2.19771647267587, "lr": 0.00029906905624647905, "optimizer_lrs": [0.019937937083098604, 0.00029906905624647905], "gradient_norm": 0.4599980115890503, "seconds": 0.30494866099991214, "nonpadding_tokens_per_second": 218062.9348624002, "ranks": [{"compute_seconds": 0.279310975000044, "nonpadding_tokens": 15415, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29643968599998516, "nonpadding_tokens": 5338, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2924837620000744, "nonpadding_tokens": 4138, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2939590329999646, "nonpadding_tokens": 2721, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29185786600010033, "nonpadding_tokens": 9324, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29236632100003135, "nonpadding_tokens": 11041, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29477944600000683, "nonpadding_tokens": 4628, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2936636660000431, "nonpadding_tokens": 13893, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003544400000237147, "prefetch": {"data_wait_seconds": 0.021384615001082352, "consumed_batches": 2278, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2828, "tokens": 206334086, "loss_tokens": 65683, "loss": 2.3800224658587457, "lr": 0.00029906840033788485, "optimizer_lrs": [0.01993789335585899, 0.00029906840033788485], "gradient_norm": 0.462697297334671, "seconds": 0.22774473399999806, "nonpadding_tokens_per_second": 288406.2294059478, "ranks": [{"compute_seconds": 0.19841339899994637, "nonpadding_tokens": 2414, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20854830500002208, "nonpadding_tokens": 10595, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20868740000003072, "nonpadding_tokens": 11378, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20588194099991597, "nonpadding_tokens": 6581, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2086950999999999, "nonpadding_tokens": 9071, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2074799819999953, "nonpadding_tokens": 13233, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20434654500002125, "nonpadding_tokens": 7697, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20521644699999797, "nonpadding_tokens": 4714, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0065588769999749275, "prefetch": {"data_wait_seconds": 0.021392178001022444, "consumed_batches": 2279, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2829, "tokens": 206393252, "loss_tokens": 59166, "loss": 2.8474356049082243, "lr": 0.0002990678093108114, "optimizer_lrs": [0.019937853954054095, 0.0002990678093108114], "gradient_norm": 0.5613117814064026, "seconds": 0.27655329100002746, "nonpadding_tokens_per_second": 213940.68313580158, "ranks": [{"compute_seconds": 0.2722741610000412, "nonpadding_tokens": 9272, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27294636999999966, "nonpadding_tokens": 2821, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26991834499995093, "nonpadding_tokens": 5488, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27022214499993424, "nonpadding_tokens": 11426, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2691625360000671, "nonpadding_tokens": 12823, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27148097499991763, "nonpadding_tokens": 4988, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2685815500000217, "nonpadding_tokens": 2518, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27407840199998645, "nonpadding_tokens": 9830, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004074380000247402, "prefetch": {"data_wait_seconds": 0.021401834000997155, "consumed_batches": 2280, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2830, "tokens": 206479387, "loss_tokens": 86135, "loss": 2.6615991612004413, "lr": 0.00029906694854836324, "optimizer_lrs": [0.019937796569890885, 0.00029906694854836324], "gradient_norm": 0.7858796715736389, "seconds": 0.30342081300000245, "nonpadding_tokens_per_second": 283879.669124739, "ranks": [{"compute_seconds": 0.2991353560000789, "nonpadding_tokens": 4574, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3011595689999922, "nonpadding_tokens": 10602, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2979445679999344, "nonpadding_tokens": 13286, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29820681199998944, "nonpadding_tokens": 8318, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29674746399996366, "nonpadding_tokens": 13797, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29994014700002936, "nonpadding_tokens": 24901, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.300786414000072, "nonpadding_tokens": 6393, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29747671099994477, "nonpadding_tokens": 4264, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003489250000257016, "prefetch": {"data_wait_seconds": 0.02141015800107482, "consumed_batches": 2281, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2831, "tokens": 206539530, "loss_tokens": 60143, "loss": 2.4283008309362684, "lr": 0.0002990663472939247, "optimizer_lrs": [0.019937756486261646, 0.0002990663472939247], "gradient_norm": 0.3927156329154968, "seconds": 0.30510503899995456, "nonpadding_tokens_per_second": 197122.27696117782, "ranks": [{"compute_seconds": 0.25633930200001487, "nonpadding_tokens": 13313, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2609967890000462, "nonpadding_tokens": 6735, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2604380919999585, "nonpadding_tokens": 4828, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26092777599990313, "nonpadding_tokens": 7801, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2572594729999764, "nonpadding_tokens": 3422, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25965489299994715, "nonpadding_tokens": 7539, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2569973419999769, "nonpadding_tokens": 10309, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.257606227999986, "nonpadding_tokens": 6196, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01711901500004842, "prefetch": {"data_wait_seconds": 0.021419431001007183, "consumed_batches": 2282, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2832, "tokens": 206584010, "loss_tokens": 44480, "loss": 2.041746641749101, "lr": 0.00029906590249962164, "optimizer_lrs": [0.01993772683330811, 0.00029906590249962164], "gradient_norm": 0.4266883134841919, "seconds": 0.24390001799997663, "nonpadding_tokens_per_second": 182369.81023922787, "ranks": [{"compute_seconds": 0.24126429299997199, "nonpadding_tokens": 2691, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2413662920000661, "nonpadding_tokens": 4022, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23837349699999777, "nonpadding_tokens": 3882, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2408660979999695, "nonpadding_tokens": 3549, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2416118319999896, "nonpadding_tokens": 14814, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24024896100002024, "nonpadding_tokens": 6771, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23716825999997582, "nonpadding_tokens": 3280, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2384213530000352, "nonpadding_tokens": 5471, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037348600005771004, "prefetch": {"data_wait_seconds": 0.021427829001098075, "consumed_batches": 2283, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2833, "tokens": 206642595, "loss_tokens": 58585, "loss": 2.240646603226082, "lr": 0.00029906531649605056, "optimizer_lrs": [0.01993768776640337, 0.00029906531649605056], "gradient_norm": 0.47902578115463257, "seconds": 0.249576114999968, "nonpadding_tokens_per_second": 234738.00768157444, "ranks": [{"compute_seconds": 0.24428906100001768, "nonpadding_tokens": 2500, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24610416500001975, "nonpadding_tokens": 6572, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.243400964999978, "nonpadding_tokens": 18137, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24418642000000546, "nonpadding_tokens": 12789, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24236879399995814, "nonpadding_tokens": 4420, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24512721400003556, "nonpadding_tokens": 959, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2464471800000183, "nonpadding_tokens": 11579, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2472562340000195, "nonpadding_tokens": 1629, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033913300001131574, "prefetch": {"data_wait_seconds": 0.02143668500112028, "consumed_batches": 2284, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2834, "tokens": 206715362, "loss_tokens": 72767, "loss": 2.425752745063009, "lr": 0.0002990645883803281, "optimizer_lrs": [0.01993763922535521, 0.0002990645883803281], "gradient_norm": 0.4439736306667328, "seconds": 0.23216463899996143, "nonpadding_tokens_per_second": 313428.43730828486, "ranks": [{"compute_seconds": 0.22812955399990642, "nonpadding_tokens": 2788, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22643258400000832, "nonpadding_tokens": 4277, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22902773399994203, "nonpadding_tokens": 2456, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22974302800002988, "nonpadding_tokens": 12569, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2298565950000011, "nonpadding_tokens": 14450, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2287971530000732, "nonpadding_tokens": 19838, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22577475699995375, "nonpadding_tokens": 10808, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22633235000000695, "nonpadding_tokens": 5581, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034333399992192426, "prefetch": {"data_wait_seconds": 0.02144518100108428, "consumed_batches": 2285, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2835, "tokens": 206781953, "loss_tokens": 66591, "loss": 2.2929222323587273, "lr": 0.0002990639218148802, "optimizer_lrs": [0.01993759478765868, 0.0002990639218148802], "gradient_norm": 1.2882874011993408, "seconds": 0.2454983859998947, "nonpadding_tokens_per_second": 271248.2191228278, "ranks": [{"compute_seconds": 0.2409858290000102, "nonpadding_tokens": 9664, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23925858799998423, "nonpadding_tokens": 9489, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24006686299992452, "nonpadding_tokens": 5642, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24074485599999207, "nonpadding_tokens": 13760, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23893193700007487, "nonpadding_tokens": 16342, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24068674700004067, "nonpadding_tokens": 1832, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24320114400006787, "nonpadding_tokens": 4841, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24001178800006073, "nonpadding_tokens": 5021, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036357600004066626, "prefetch": {"data_wait_seconds": 0.021453782001074728, "consumed_batches": 2286, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2836, "tokens": 206865297, "loss_tokens": 83344, "loss": 2.4575619945646956, "lr": 0.00029906308722118467, "optimizer_lrs": [0.01993753914807898, 0.00029906308722118467], "gradient_norm": 0.6442067623138428, "seconds": 0.2783022560000745, "nonpadding_tokens_per_second": 299472.95863809914, "ranks": [{"compute_seconds": 0.2751850919999015, "nonpadding_tokens": 7881, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2760769789999813, "nonpadding_tokens": 2502, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2728041379999695, "nonpadding_tokens": 9447, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.273438877999979, "nonpadding_tokens": 8802, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27199841499998456, "nonpadding_tokens": 19899, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27500161300008585, "nonpadding_tokens": 13168, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27199246799989396, "nonpadding_tokens": 14422, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27232201799995437, "nonpadding_tokens": 7223, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033801799997945636, "prefetch": {"data_wait_seconds": 0.02146152900111531, "consumed_batches": 2287, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2837, "tokens": 206922558, "loss_tokens": 57261, "loss": 2.4740844008138176, "lr": 0.0002990625136037717, "optimizer_lrs": [0.019937500906918114, 0.0002990625136037717], "gradient_norm": 0.38071322441101074, "seconds": 0.2868780539999989, "nonpadding_tokens_per_second": 199600.4894818487, "ranks": [{"compute_seconds": 0.28330015500000627, "nonpadding_tokens": 1414, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2844650959999626, "nonpadding_tokens": 14276, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28043022300005305, "nonpadding_tokens": 4668, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2811247070000036, "nonpadding_tokens": 8605, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27951719899999716, "nonpadding_tokens": 2776, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2810550680000006, "nonpadding_tokens": 13031, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2793811569999889, "nonpadding_tokens": 6664, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2797661600000083, "nonpadding_tokens": 5827, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036974799991185137, "prefetch": {"data_wait_seconds": 0.021470549001108452, "consumed_batches": 2288, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2838, "tokens": 206956301, "loss_tokens": 33743, "loss": 2.3874660114690456, "lr": 0.00029906217549820424, "optimizer_lrs": [0.01993747836654695, 0.00029906217549820424], "gradient_norm": 0.47277703881263733, "seconds": 0.2823302130000229, "nonpadding_tokens_per_second": 119516.07885478859, "ranks": [{"compute_seconds": 0.2775967689999561, "nonpadding_tokens": 2154, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27966146899996147, "nonpadding_tokens": 4315, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2769430079999893, "nonpadding_tokens": 3350, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27695914399998856, "nonpadding_tokens": 6514, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2799437459999581, "nonpadding_tokens": 884, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2785172919999468, "nonpadding_tokens": 4218, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2756137050000689, "nonpadding_tokens": 5535, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2764076280000154, "nonpadding_tokens": 6773, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039866900010565587, "prefetch": {"data_wait_seconds": 0.021479134001040734, "consumed_batches": 2289, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2839, "tokens": 207017373, "loss_tokens": 61072, "loss": 2.5058253475406076, "lr": 0.0002990615634010168, "optimizer_lrs": [0.01993743756006779, 0.0002990615634010168], "gradient_norm": 1.064157247543335, "seconds": 0.34612402999994174, "nonpadding_tokens_per_second": 176445.42044656732, "ranks": [{"compute_seconds": 0.34122312499994223, "nonpadding_tokens": 10505, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3438491190000832, "nonpadding_tokens": 12500, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3407660969999142, "nonpadding_tokens": 3413, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34147056599999814, "nonpadding_tokens": 9291, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33978050899997925, "nonpadding_tokens": 4862, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3411691040000733, "nonpadding_tokens": 7475, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33951973800003543, "nonpadding_tokens": 1453, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34001782799998637, "nonpadding_tokens": 11573, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033849800001917174, "prefetch": {"data_wait_seconds": 0.021488913001078913, "consumed_batches": 2290, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2840, "tokens": 207081430, "loss_tokens": 64057, "loss": 2.438078830182494, "lr": 0.00029906092117270097, "optimizer_lrs": [0.019937394744846734, 0.00029906092117270097], "gradient_norm": 0.4585670530796051, "seconds": 0.26171497400002863, "nonpadding_tokens_per_second": 244758.63578211993, "ranks": [{"compute_seconds": 0.2316363200000069, "nonpadding_tokens": 9160, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2421700969999847, "nonpadding_tokens": 7601, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.239009941000063, "nonpadding_tokens": 9549, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2399242349999895, "nonpadding_tokens": 1762, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24256596800000807, "nonpadding_tokens": 14742, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24081133599997884, "nonpadding_tokens": 10654, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24249209700008123, "nonpadding_tokens": 5242, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23833281200006695, "nonpadding_tokens": 5347, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.010462867999990522, "prefetch": {"data_wait_seconds": 0.0214986240011541, "consumed_batches": 2291, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2841, "tokens": 207146568, "loss_tokens": 65138, "loss": 2.2582236463354723, "lr": 0.00029906026788191205, "optimizer_lrs": [0.019937351192127473, 0.00029906026788191205], "gradient_norm": 0.42478424310684204, "seconds": 0.25840453699993304, "nonpadding_tokens_per_second": 252077.61735242628, "ranks": [{"compute_seconds": 0.2505950480000365, "nonpadding_tokens": 4423, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25599225399992065, "nonpadding_tokens": 4073, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2533811929999956, "nonpadding_tokens": 14983, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25603023899998334, "nonpadding_tokens": 6163, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2522711440000194, "nonpadding_tokens": 9278, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2546028929999693, "nonpadding_tokens": 15373, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25208446899989667, "nonpadding_tokens": 5111, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25282382800003234, "nonpadding_tokens": 5734, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039719799997328664, "prefetch": {"data_wait_seconds": 0.021507180001165693, "consumed_batches": 2292, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2842, "tokens": 207218354, "loss_tokens": 71786, "loss": 2.23690051855515, "lr": 0.00029905954765390883, "optimizer_lrs": [0.019937303176927258, 0.00029905954765390883], "gradient_norm": 0.4362759292125702, "seconds": 0.2835173759999634, "nonpadding_tokens_per_second": 253197.88512718622, "ranks": [{"compute_seconds": 0.27736524099998405, "nonpadding_tokens": 7610, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2780768680000847, "nonpadding_tokens": 10912, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2808507519999921, "nonpadding_tokens": 13999, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27763953800001673, "nonpadding_tokens": 2000, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.275627198000052, "nonpadding_tokens": 5985, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27856144099996527, "nonpadding_tokens": 6019, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27544300800002475, "nonpadding_tokens": 7818, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27616132799994375, "nonpadding_tokens": 17443, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003736099999969156, "prefetch": {"data_wait_seconds": 0.02151548800118519, "consumed_batches": 2293, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2843, "tokens": 207322172, "loss_tokens": 103818, "loss": 2.1548871883970024, "lr": 0.0002990585055629542, "optimizer_lrs": [0.019937233704196946, 0.0002990585055629542], "gradient_norm": 0.40080079436302185, "seconds": 0.35669606600004045, "nonpadding_tokens_per_second": 291054.5136205349, "ranks": [{"compute_seconds": 0.35425475599993206, "nonpadding_tokens": 19143, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35114731900000606, "nonpadding_tokens": 16344, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3517528800000491, "nonpadding_tokens": 2761, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35231624100003955, "nonpadding_tokens": 12814, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.35026911800002836, "nonpadding_tokens": 10851, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3533959330000016, "nonpadding_tokens": 6395, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3507136430000628, "nonpadding_tokens": 18394, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35150837700007287, "nonpadding_tokens": 17116, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038464399995064014, "prefetch": {"data_wait_seconds": 0.02152412700122852, "consumed_batches": 2294, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2844, "tokens": 207409833, "loss_tokens": 87661, "loss": 2.063986906948358, "lr": 0.0002990576252029065, "optimizer_lrs": [0.0199371750135271, 0.0002990576252029065], "gradient_norm": 0.39964354038238525, "seconds": 0.350802824000084, "nonpadding_tokens_per_second": 249886.81390996728, "ranks": [{"compute_seconds": 0.346900871999992, "nonpadding_tokens": 10591, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3475981020000063, "nonpadding_tokens": 12760, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.344540717999962, "nonpadding_tokens": 17435, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34694147000004705, "nonpadding_tokens": 11334, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.34339045600006557, "nonpadding_tokens": 3915, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3460279820000096, "nonpadding_tokens": 6945, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3471532460000617, "nonpadding_tokens": 12752, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34842543100000967, "nonpadding_tokens": 11929, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003854349999983242, "prefetch": {"data_wait_seconds": 0.021534373001216522, "consumed_batches": 2295, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2845, "tokens": 207466041, "loss_tokens": 56208, "loss": 2.3988277914175917, "lr": 0.0002990570605026769, "optimizer_lrs": [0.019937137366845128, 0.0002990570605026769], "gradient_norm": 0.5317623615264893, "seconds": 0.24598527800003467, "nonpadding_tokens_per_second": 228501.47966982023, "ranks": [{"compute_seconds": 0.24107616000003418, "nonpadding_tokens": 5903, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23988594000002195, "nonpadding_tokens": 12869, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2404986709999548, "nonpadding_tokens": 2218, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2428019490000679, "nonpadding_tokens": 6445, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.239625718999946, "nonpadding_tokens": 3688, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24105452000003424, "nonpadding_tokens": 2421, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23947339699998338, "nonpadding_tokens": 13956, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24369277300002068, "nonpadding_tokens": 8708, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003679709999460101, "prefetch": {"data_wait_seconds": 0.021543666001164752, "consumed_batches": 2296, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2846, "tokens": 207528317, "loss_tokens": 62276, "loss": 2.3161355297385833, "lr": 0.000299056434642752, "optimizer_lrs": [0.019937095642850133, 0.000299056434642752], "gradient_norm": 0.6315670013427734, "seconds": 0.2531003279999595, "nonpadding_tokens_per_second": 246052.62463354043, "ranks": [{"compute_seconds": 0.24713626599998406, "nonpadding_tokens": 7077, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2496580140000333, "nonpadding_tokens": 6118, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24644993199990495, "nonpadding_tokens": 2960, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25024011400000745, "nonpadding_tokens": 6568, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2507943210000576, "nonpadding_tokens": 14397, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24855450999996265, "nonpadding_tokens": 17539, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24550030499995046, "nonpadding_tokens": 4011, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2465640179999582, "nonpadding_tokens": 3606, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036821100002271123, "prefetch": {"data_wait_seconds": 0.021551885001215965, "consumed_batches": 2297, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2847, "tokens": 207592241, "loss_tokens": 63924, "loss": 2.4046626658219137, "lr": 0.0002990557920056043, "optimizer_lrs": [0.01993705280037362, 0.0002990557920056043], "gradient_norm": 0.4391494691371918, "seconds": 0.27533242300000893, "nonpadding_tokens_per_second": 232170.25914887592, "ranks": [{"compute_seconds": 0.27084084700004496, "nonpadding_tokens": 15678, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27311121800005367, "nonpadding_tokens": 2772, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2698627969999734, "nonpadding_tokens": 5602, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2725040049999734, "nonpadding_tokens": 17120, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26890339099998073, "nonpadding_tokens": 4696, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2706524840000384, "nonpadding_tokens": 3515, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27278499199996986, "nonpadding_tokens": 4537, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2695350950000375, "nonpadding_tokens": 10004, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003473049999911382, "prefetch": {"data_wait_seconds": 0.02156045300121434, "consumed_batches": 2298, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2848, "tokens": 207685351, "loss_tokens": 93110, "loss": 2.5049947642573303, "lr": 0.00029905485556738965, "optimizer_lrs": [0.019936990371159313, 0.00029905485556738965], "gradient_norm": 0.40057510137557983, "seconds": 0.3116882069999747, "nonpadding_tokens_per_second": 298728.0170019636, "ranks": [{"compute_seconds": 0.30764064300001337, "nonpadding_tokens": 12883, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30602460000000065, "nonpadding_tokens": 18122, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30685077499992985, "nonpadding_tokens": 7321, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30937558899995565, "nonpadding_tokens": 17726, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3054860510000026, "nonpadding_tokens": 14589, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.308567958000026, "nonpadding_tokens": 11073, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3093953100000135, "nonpadding_tokens": 6469, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3060718620000671, "nonpadding_tokens": 4927, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003917220000175803, "prefetch": {"data_wait_seconds": 0.021569069001273533, "consumed_batches": 2299, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2849, "tokens": 207756843, "loss_tokens": 71492, "loss": 2.276796267414536, "lr": 0.00029905413623468926, "optimizer_lrs": [0.01993694241564595, 0.00029905413623468926], "gradient_norm": 0.5002405047416687, "seconds": 0.3304776549999815, "nonpadding_tokens_per_second": 216329.3006905535, "ranks": [{"compute_seconds": 0.29915802300001815, "nonpadding_tokens": 7965, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30872998000006646, "nonpadding_tokens": 7556, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3081023099999811, "nonpadding_tokens": 6148, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3063889710000467, "nonpadding_tokens": 5441, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30463597699997536, "nonpadding_tokens": 3104, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3075033860000076, "nonpadding_tokens": 10621, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3084138049999865, "nonpadding_tokens": 5149, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3089599200000066, "nonpadding_tokens": 25508, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.007973168000035002, "prefetch": {"data_wait_seconds": 0.021578280001293138, "consumed_batches": 2300, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2850, "tokens": 207808038, "loss_tokens": 51195, "loss": 2.238523506934271, "lr": 0.00029905362095724084, "optimizer_lrs": [0.019936908063816056, 0.00029905362095724084], "gradient_norm": 0.39797985553741455, "seconds": 0.4584235049999279, "nonpadding_tokens_per_second": 111676.21084352568, "ranks": [{"compute_seconds": 0.4558557130000054, "nonpadding_tokens": 13507, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4559163929999386, "nonpadding_tokens": 1560, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4549296290000484, "nonpadding_tokens": 3613, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4535371240000359, "nonpadding_tokens": 4326, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.4516090650000706, "nonpadding_tokens": 17287, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4546978280000076, "nonpadding_tokens": 1468, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.45166426400010096, "nonpadding_tokens": 3903, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4561220919999869, "nonpadding_tokens": 5531, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033780100000058155, "prefetch": {"data_wait_seconds": 0.02158648700124388, "consumed_batches": 2301, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2851, "tokens": 207861725, "loss_tokens": 53687, "loss": 2.3676271141058356, "lr": 0.000299053080447627, "optimizer_lrs": [0.0199368720298418, 0.000299053080447627], "gradient_norm": 0.4255993962287903, "seconds": 0.25475657699996646, "nonpadding_tokens_per_second": 210738.4258032603, "ranks": [{"compute_seconds": 0.252442659000053, "nonpadding_tokens": 10088, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2522846169999866, "nonpadding_tokens": 10542, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24930043200004093, "nonpadding_tokens": 2135, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24991015700004482, "nonpadding_tokens": 6019, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24830046499994296, "nonpadding_tokens": 5924, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2511184139999614, "nonpadding_tokens": 9310, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2521404020000091, "nonpadding_tokens": 4934, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24907979900001465, "nonpadding_tokens": 4735, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041419399997266737, "prefetch": {"data_wait_seconds": 0.021597558001261064, "consumed_batches": 2302, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2852, "tokens": 207925306, "loss_tokens": 63581, "loss": 2.3071974626853935, "lr": 0.0002990524401283855, "optimizer_lrs": [0.019936829341892368, 0.0002990524401283855], "gradient_norm": 0.39911630749702454, "seconds": 0.2242816780000112, "nonpadding_tokens_per_second": 283487.2672925018, "ranks": [{"compute_seconds": 0.2202422600000773, "nonpadding_tokens": 2435, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2218654380000089, "nonpadding_tokens": 3233, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22118655799999942, "nonpadding_tokens": 14892, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2219426519999388, "nonpadding_tokens": 16901, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21821256800001265, "nonpadding_tokens": 3048, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22074556199993367, "nonpadding_tokens": 7230, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21814000599999872, "nonpadding_tokens": 4106, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21867288900000403, "nonpadding_tokens": 11736, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004031279999026083, "prefetch": {"data_wait_seconds": 0.021606317001285902, "consumed_batches": 2303, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2853, "tokens": 208008091, "loss_tokens": 82785, "loss": 2.1818426496345955, "lr": 0.00029905160608396865, "optimizer_lrs": [0.019936773738931244, 0.00029905160608396865], "gradient_norm": 0.3865039646625519, "seconds": 0.2816926749999311, "nonpadding_tokens_per_second": 293884.106145182, "ranks": [{"compute_seconds": 0.2759937839999793, "nonpadding_tokens": 13250, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2791367590000391, "nonpadding_tokens": 2410, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2790590270000166, "nonpadding_tokens": 8540, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2794407809999484, "nonpadding_tokens": 3256, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27515546900008303, "nonpadding_tokens": 17790, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2771941889999425, "nonpadding_tokens": 19945, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2735672280000472, "nonpadding_tokens": 6841, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2758510450000813, "nonpadding_tokens": 10753, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003680309999936071, "prefetch": {"data_wait_seconds": 0.021614354001215474, "consumed_batches": 2304, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2854, "tokens": 208152909, "loss_tokens": 144818, "loss": 2.2756477095388696, "lr": 0.00029905014618872023, "optimizer_lrs": [0.01993667641258135, 0.00029905014618872023], "gradient_norm": 0.549039363861084, "seconds": 0.23417993299995032, "nonpadding_tokens_per_second": 618404.8229274654, "ranks": [{"compute_seconds": 0.2298393410000017, "nonpadding_tokens": 3461, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23167797799999335, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22885001899999224, "nonpadding_tokens": 18803, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2295506769999065, "nonpadding_tokens": 18575, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23183722600003875, "nonpadding_tokens": 27534, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23025657499999852, "nonpadding_tokens": 17835, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23132202099998267, "nonpadding_tokens": 8434, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22866925400001037, "nonpadding_tokens": 17408, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003750550000631847, "prefetch": {"data_wait_seconds": 0.02162273400126935, "consumed_batches": 2305, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2855, "tokens": 208234791, "loss_tokens": 81882, "loss": 2.4414498531423265, "lr": 0.0002990493202496351, "optimizer_lrs": [0.019936621349975675, 0.0002990493202496351], "gradient_norm": 0.5506372451782227, "seconds": 0.2892346249999491, "nonpadding_tokens_per_second": 283098.88555014605, "ranks": [{"compute_seconds": 0.2841965509999227, "nonpadding_tokens": 6188, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2869792740000321, "nonpadding_tokens": 10210, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28616960499994093, "nonpadding_tokens": 11315, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.284010606000038, "nonpadding_tokens": 18238, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2830724159999818, "nonpadding_tokens": 15968, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2854617559999042, "nonpadding_tokens": 10052, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28614758800006257, "nonpadding_tokens": 5879, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2834689899999603, "nonpadding_tokens": 4032, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003790270000081364, "prefetch": {"data_wait_seconds": 0.02163157200125454, "consumed_batches": 2306, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2856, "tokens": 208268487, "loss_tokens": 33696, "loss": 2.357745309161325, "lr": 0.0002990489802561167, "optimizer_lrs": [0.019936598683741112, 0.0002990489802561167], "gradient_norm": 0.488058477640152, "seconds": 0.2170297569999775, "nonpadding_tokens_per_second": 155259.81536257025, "ranks": [{"compute_seconds": 0.2142778030000727, "nonpadding_tokens": 5432, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2112900829999944, "nonpadding_tokens": 5056, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21129574100007176, "nonpadding_tokens": 4840, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21131338800000776, "nonpadding_tokens": 2909, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21002458699990711, "nonpadding_tokens": 5066, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2127413760000536, "nonpadding_tokens": 2363, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2103380679999418, "nonpadding_tokens": 3716, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21477473000004466, "nonpadding_tokens": 4314, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003453049999961877, "prefetch": {"data_wait_seconds": 0.021640423001215936, "consumed_batches": 2307, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2857, "tokens": 208314210, "loss_tokens": 45723, "loss": 1.9938676240622881, "lr": 0.00029904851881299723, "optimizer_lrs": [0.019936567920866483, 0.00029904851881299723], "gradient_norm": 0.6809918284416199, "seconds": 0.29350231099999746, "nonpadding_tokens_per_second": 155784.12259929496, "ranks": [{"compute_seconds": 0.28753895700003795, "nonpadding_tokens": 2517, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2899071309999499, "nonpadding_tokens": 6284, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2869145550000667, "nonpadding_tokens": 2294, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2877088799999683, "nonpadding_tokens": 5972, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2911753130000534, "nonpadding_tokens": 14202, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28895441199995275, "nonpadding_tokens": 2849, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2854849909999757, "nonpadding_tokens": 8702, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28704633999996076, "nonpadding_tokens": 2903, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003941869999835035, "prefetch": {"data_wait_seconds": 0.021649445001187928, "consumed_batches": 2308, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2858, "tokens": 208351410, "loss_tokens": 37200, "loss": 2.3834677419354837, "lr": 0.0002990481433029453, "optimizer_lrs": [0.019936542886863022, 0.0002990481433029453], "gradient_norm": 0.4599321484565735, "seconds": 0.2387861070000099, "nonpadding_tokens_per_second": 155787.95796523648, "ranks": [{"compute_seconds": 0.2354903100000456, "nonpadding_tokens": 2165, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23615066900003967, "nonpadding_tokens": 2342, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2328498929999796, "nonpadding_tokens": 3472, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2353270710000288, "nonpadding_tokens": 2645, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23608902200010107, "nonpadding_tokens": 4819, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23505532699994092, "nonpadding_tokens": 7662, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.231843562999984, "nonpadding_tokens": 7421, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23648117100003674, "nonpadding_tokens": 6674, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000365811000051508, "prefetch": {"data_wait_seconds": 0.021658042001106992, "consumed_batches": 2309, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2859, "tokens": 208417194, "loss_tokens": 65784, "loss": 2.1153101438039643, "lr": 0.00029904747907511333, "optimizer_lrs": [0.01993649860500756, 0.00029904747907511333], "gradient_norm": 0.7262071371078491, "seconds": 0.2938353540000662, "nonpadding_tokens_per_second": 223880.47967837518, "ranks": [{"compute_seconds": 0.28979208200007633, "nonpadding_tokens": 5430, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2885768609999104, "nonpadding_tokens": 3615, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2879456340000388, "nonpadding_tokens": 10941, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2890015849999372, "nonpadding_tokens": 7401, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2872371780000549, "nonpadding_tokens": 12013, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2904335999999148, "nonpadding_tokens": 9261, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2914939739999909, "nonpadding_tokens": 7462, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2875806030000376, "nonpadding_tokens": 9661, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004047340000852273, "prefetch": {"data_wait_seconds": 0.021666563001147665, "consumed_batches": 2310, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2860, "tokens": 208467570, "loss_tokens": 50376, "loss": 2.682008483702557, "lr": 0.00029904697026736523, "optimizer_lrs": [0.019936464684491017, 0.00029904697026736523], "gradient_norm": 0.5006626844406128, "seconds": 0.2875688550000177, "nonpadding_tokens_per_second": 175178.91497671712, "ranks": [{"compute_seconds": 0.28270906399995965, "nonpadding_tokens": 11267, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2819958080000333, "nonpadding_tokens": 4658, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2811394940000582, "nonpadding_tokens": 3489, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2812040310000157, "nonpadding_tokens": 4453, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28007251600001837, "nonpadding_tokens": 5713, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28282562399999733, "nonpadding_tokens": 5698, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2852461759999869, "nonpadding_tokens": 4234, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28097251799999867, "nonpadding_tokens": 10864, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036224200005108287, "prefetch": {"data_wait_seconds": 0.021674600001190925, "consumed_batches": 2311, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2861, "tokens": 208536666, "loss_tokens": 69096, "loss": 2.4072042882366564, "lr": 0.0002990462721636615, "optimizer_lrs": [0.0199364181442441, 0.0002990462721636615], "gradient_norm": 0.4330083131790161, "seconds": 0.3142856160000065, "nonpadding_tokens_per_second": 219850.97784430126, "ranks": [{"compute_seconds": 0.3115759969999772, "nonpadding_tokens": 13758, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31107729099994685, "nonpadding_tokens": 7228, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30782317899991085, "nonpadding_tokens": 3846, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30824872900006994, "nonpadding_tokens": 4151, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30698816999995415, "nonpadding_tokens": 6339, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3095720200000187, "nonpadding_tokens": 8105, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31111026700000366, "nonpadding_tokens": 9882, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3119300270000167, "nonpadding_tokens": 15787, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000391770999954133, "prefetch": {"data_wait_seconds": 0.021685608001192236, "consumed_batches": 2312, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2862, "tokens": 208607883, "loss_tokens": 71217, "loss": 2.2607944732297063, "lr": 0.0002990455523641408, "optimizer_lrs": [0.01993637015760939, 0.0002990455523641408], "gradient_norm": 0.5106834173202515, "seconds": 0.3770822840000392, "nonpadding_tokens_per_second": 188863.28799258202, "ranks": [{"compute_seconds": 0.37459548699996503, "nonpadding_tokens": 17670, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37059467500000665, "nonpadding_tokens": 11016, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3726801580000938, "nonpadding_tokens": 5242, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.37400815800003784, "nonpadding_tokens": 5663, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.37292510100007803, "nonpadding_tokens": 8359, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.371561679000024, "nonpadding_tokens": 5041, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3724746079999477, "nonpadding_tokens": 15504, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3725445730000274, "nonpadding_tokens": 2722, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037824399998953595, "prefetch": {"data_wait_seconds": 0.021695638001190787, "consumed_batches": 2313, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2863, "tokens": 208666256, "loss_tokens": 58373, "loss": 2.2074879706799377, "lr": 0.0002990449621788401, "optimizer_lrs": [0.019936330811922677, 0.0002990449621788401], "gradient_norm": 0.3916245400905609, "seconds": 0.30852523900000506, "nonpadding_tokens_per_second": 189200.08032146454, "ranks": [{"compute_seconds": 0.3018850260000363, "nonpadding_tokens": 2419, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30191841299995303, "nonpadding_tokens": 11472, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.300955756999997, "nonpadding_tokens": 2801, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30122923300007187, "nonpadding_tokens": 8158, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3059672340000361, "nonpadding_tokens": 12020, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3032364220000545, "nonpadding_tokens": 7248, "peak_allocated_bytes": 14994384384, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3055511039999601, "nonpadding_tokens": 3630, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2996696530000236, "nonpadding_tokens": 10625, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004497600000377133, "prefetch": {"data_wait_seconds": 0.02170658400127934, "consumed_batches": 2314, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2864, "tokens": 208761980, "loss_tokens": 95724, "loss": 2.068084088107476, "lr": 0.00029904399395956274, "optimizer_lrs": [0.01993626626397085, 0.00029904399395956274], "gradient_norm": 0.4816988408565521, "seconds": 0.3939617470000485, "nonpadding_tokens_per_second": 242977.90516191465, "ranks": [{"compute_seconds": 0.38998268700004246, "nonpadding_tokens": 10973, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3846227660000068, "nonpadding_tokens": 20027, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3900622319999911, "nonpadding_tokens": 9134, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.390736453000045, "nonpadding_tokens": 9015, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.38850786399996196, "nonpadding_tokens": 7300, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.39140523499997926, "nonpadding_tokens": 21497, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3886136250000618, "nonpadding_tokens": 14125, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.39069791800000075, "nonpadding_tokens": 3653, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042125300001316646, "prefetch": {"data_wait_seconds": 0.0217162380012752, "consumed_batches": 2315, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2865, "tokens": 208822669, "loss_tokens": 60689, "loss": 1.9768281988910676, "lr": 0.00029904337985551567, "optimizer_lrs": [0.019936225323701047, 0.00029904337985551567], "gradient_norm": 0.4402221143245697, "seconds": 0.36095487299996876, "nonpadding_tokens_per_second": 168134.59116260568, "ranks": [{"compute_seconds": 0.3189270389999592, "nonpadding_tokens": 2866, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3218980059999694, "nonpadding_tokens": 6248, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31801564199997756, "nonpadding_tokens": 6740, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32088759300006586, "nonpadding_tokens": 9343, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3212488880000137, "nonpadding_tokens": 4154, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31850926799995705, "nonpadding_tokens": 5614, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3169819779999443, "nonpadding_tokens": 10514, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.318160128000045, "nonpadding_tokens": 15210, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.037190312000007, "prefetch": {"data_wait_seconds": 0.021726528001252632, "consumed_batches": 2316, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2866, "tokens": 208907563, "loss_tokens": 84894, "loss": 2.379318252762268, "lr": 0.0002990425204945989, "optimizer_lrs": [0.01993616803297326, 0.0002990425204945989], "gradient_norm": 0.4009626805782318, "seconds": 0.2578857830000061, "nonpadding_tokens_per_second": 329192.2455453777, "ranks": [{"compute_seconds": 0.25415590800002974, "nonpadding_tokens": 10163, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2536077759999671, "nonpadding_tokens": 5803, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2506045779999795, "nonpadding_tokens": 11763, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2511502519998885, "nonpadding_tokens": 3684, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2553648800000019, "nonpadding_tokens": 15012, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2525278929999786, "nonpadding_tokens": 14850, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2547153859999298, "nonpadding_tokens": 10764, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25014370599990343, "nonpadding_tokens": 12855, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036076399999274145, "prefetch": {"data_wait_seconds": 0.0217355190012114, "consumed_batches": 2317, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2867, "tokens": 209010716, "loss_tokens": 103153, "loss": 2.2635950966040737, "lr": 0.0002990414757849809, "optimizer_lrs": [0.019936098385665395, 0.0002990414757849809], "gradient_norm": 0.3905603289604187, "seconds": 0.30839103499999965, "nonpadding_tokens_per_second": 334487.6740661418, "ranks": [{"compute_seconds": 0.30422894299999825, "nonpadding_tokens": 9987, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30320151799992345, "nonpadding_tokens": 15948, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.305584983000017, "nonpadding_tokens": 5353, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30604448400004003, "nonpadding_tokens": 18585, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3025277830000732, "nonpadding_tokens": 2946, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3049454509999805, "nonpadding_tokens": 23517, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3019843999999239, "nonpadding_tokens": 7296, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3032189140000128, "nonpadding_tokens": 19521, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003471509999144473, "prefetch": {"data_wait_seconds": 0.021744356001207166, "consumed_batches": 2318, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2868, "tokens": 209075287, "loss_tokens": 64571, "loss": 2.0653263461925633, "lr": 0.0002990408215360377, "optimizer_lrs": [0.01993605476906918, 0.0002990408215360377], "gradient_norm": 0.505284309387207, "seconds": 0.24574026800007687, "nonpadding_tokens_per_second": 262761.1686334606, "ranks": [{"compute_seconds": 0.24026942000000417, "nonpadding_tokens": 7040, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.243028656999968, "nonpadding_tokens": 1887, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24197581700002502, "nonpadding_tokens": 7675, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2428430439999829, "nonpadding_tokens": 3701, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24338749699995788, "nonpadding_tokens": 5336, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.241530852999972, "nonpadding_tokens": 13944, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23888516400006665, "nonpadding_tokens": 17316, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23967081700004655, "nonpadding_tokens": 7672, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003694060000043464, "prefetch": {"data_wait_seconds": 0.021752789001155293, "consumed_batches": 2319, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2869, "tokens": 209203400, "loss_tokens": 128113, "loss": 1.666027745037584, "lr": 0.0002990395228058326, "optimizer_lrs": [0.01993596818705551, 0.0002990395228058326], "gradient_norm": 0.7567210793495178, "seconds": 0.27718147499990664, "nonpadding_tokens_per_second": 462198.99796710134, "ranks": [{"compute_seconds": 0.2711793739999848, "nonpadding_tokens": 9753, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2745933360000663, "nonpadding_tokens": 11890, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2712248080000563, "nonpadding_tokens": 12333, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27232389999994666, "nonpadding_tokens": 11896, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2698870450000186, "nonpadding_tokens": 20145, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2722281750000093, "nonpadding_tokens": 21061, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2696817779999492, "nonpadding_tokens": 19987, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2700962909999589, "nonpadding_tokens": 21048, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004319270000223696, "prefetch": {"data_wait_seconds": 0.021759646001171973, "consumed_batches": 2320, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2870, "tokens": 209279802, "loss_tokens": 76402, "loss": 2.221786978744012, "lr": 0.0002990387478729526, "optimizer_lrs": [0.019935916524863508, 0.0002990387478729526], "gradient_norm": 0.6824187636375427, "seconds": 0.35944385699997383, "nonpadding_tokens_per_second": 212556.1433645688, "ranks": [{"compute_seconds": 0.3563266429999885, "nonpadding_tokens": 18336, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3569434269999192, "nonpadding_tokens": 6773, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3543321060000153, "nonpadding_tokens": 10698, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35503450600003816, "nonpadding_tokens": 4899, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3536454060000551, "nonpadding_tokens": 3141, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35625888399999894, "nonpadding_tokens": 7958, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35325425000007726, "nonpadding_tokens": 18240, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35416331099997933, "nonpadding_tokens": 6357, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005061169999862614, "prefetch": {"data_wait_seconds": 0.021779344001132017, "consumed_batches": 2321, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2871, "tokens": 209357230, "loss_tokens": 77428, "loss": 2.57279242489797, "lr": 0.0002990379622158506, "optimizer_lrs": [0.019935864147723376, 0.0002990379622158506], "gradient_norm": 0.5546153783798218, "seconds": 0.36182853900004375, "nonpadding_tokens_per_second": 213990.85935559837, "ranks": [{"compute_seconds": 0.358546540999896, "nonpadding_tokens": 13496, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3591786110000612, "nonpadding_tokens": 10320, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35573873399994227, "nonpadding_tokens": 8350, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3565744559999757, "nonpadding_tokens": 2748, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.35946093200004725, "nonpadding_tokens": 16808, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3577467650000017, "nonpadding_tokens": 3799, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3548027200000661, "nonpadding_tokens": 3273, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3554463199999418, "nonpadding_tokens": 18634, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003440109999246488, "prefetch": {"data_wait_seconds": 0.02178813400117008, "consumed_batches": 2322, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2872, "tokens": 209411206, "loss_tokens": 53976, "loss": 2.113192813704239, "lr": 0.0002990374143355662, "optimizer_lrs": [0.01993582762237108, 0.0002990374143355662], "gradient_norm": 0.4033614993095398, "seconds": 0.29965949700010697, "nonpadding_tokens_per_second": 180124.44304403517, "ranks": [{"compute_seconds": 0.29615349099992727, "nonpadding_tokens": 15155, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29348288300002423, "nonpadding_tokens": 5341, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2938165159999926, "nonpadding_tokens": 5611, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2943024339999738, "nonpadding_tokens": 4902, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2972231249999595, "nonpadding_tokens": 4799, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2958252829999992, "nonpadding_tokens": 3872, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2925763590000088, "nonpadding_tokens": 10248, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29365418999998383, "nonpadding_tokens": 4048, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003794189999553055, "prefetch": {"data_wait_seconds": 0.021798061001121027, "consumed_batches": 2323, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2873, "tokens": 209463524, "loss_tokens": 52318, "loss": 2.4299019338468595, "lr": 0.0002990368831363964, "optimizer_lrs": [0.019935792209093094, 0.0002990368831363964], "gradient_norm": 0.4250178635120392, "seconds": 0.2759223750000501, "nonpadding_tokens_per_second": 189611.29919235618, "ranks": [{"compute_seconds": 0.2718862229999104, "nonpadding_tokens": 10901, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27285103299993807, "nonpadding_tokens": 7349, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26945370800001456, "nonpadding_tokens": 3613, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2703567579999344, "nonpadding_tokens": 12749, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26919298400002845, "nonpadding_tokens": 9255, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27030821200003174, "nonpadding_tokens": 4178, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27251095500002975, "nonpadding_tokens": 2483, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2736221679999744, "nonpadding_tokens": 1790, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003531229999680363, "prefetch": {"data_wait_seconds": 0.02180753700118032, "consumed_batches": 2324, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2874, "tokens": 209528540, "loss_tokens": 65016, "loss": 2.353136872077642, "lr": 0.0002990362228074494, "optimizer_lrs": [0.019935748187163295, 0.0002990362228074494], "gradient_norm": 0.4500260651111603, "seconds": 0.29908642500004134, "nonpadding_tokens_per_second": 217381.98248212374, "ranks": [{"compute_seconds": 0.29303614900004504, "nonpadding_tokens": 7434, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2927853190000178, "nonpadding_tokens": 4801, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2918501079999487, "nonpadding_tokens": 8800, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2925329669999428, "nonpadding_tokens": 11465, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.291327334000016, "nonpadding_tokens": 12329, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29412262900007136, "nonpadding_tokens": 4527, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29121284699999705, "nonpadding_tokens": 10868, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29667187100005776, "nonpadding_tokens": 4792, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003412570000591586, "prefetch": {"data_wait_seconds": 0.02181609200120249, "consumed_batches": 2325, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2875, "tokens": 209611076, "loss_tokens": 82536, "loss": 2.4065814841402537, "lr": 0.00029903538421345035, "optimizer_lrs": [0.019935692280896693, 0.00029903538421345035], "gradient_norm": 0.4311347007751465, "seconds": 0.264867671999923, "nonpadding_tokens_per_second": 311612.20762352605, "ranks": [{"compute_seconds": 0.26061903599998004, "nonpadding_tokens": 2489, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25877457100000356, "nonpadding_tokens": 12935, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2596193240000275, "nonpadding_tokens": 14577, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.262532694000015, "nonpadding_tokens": 1922, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2581840100000363, "nonpadding_tokens": 13375, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.261174780000033, "nonpadding_tokens": 13180, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25875219099998503, "nonpadding_tokens": 14958, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2590851869999824, "nonpadding_tokens": 9100, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003734470000154033, "prefetch": {"data_wait_seconds": 0.02182447000116383, "consumed_batches": 2326, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2876, "tokens": 209671770, "loss_tokens": 60694, "loss": 2.302816176228293, "lr": 0.0002990347673097925, "optimizer_lrs": [0.01993565115398617, 0.0002990347673097925], "gradient_norm": 0.4493601620197296, "seconds": 0.3672512770000367, "nonpadding_tokens_per_second": 165265.59280008695, "ranks": [{"compute_seconds": 0.36421146599991516, "nonpadding_tokens": 8990, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3646827210000083, "nonpadding_tokens": 10372, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.363590461000058, "nonpadding_tokens": 3060, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3619571919999771, "nonpadding_tokens": 9700, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3602039600000353, "nonpadding_tokens": 5375, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3630705460000172, "nonpadding_tokens": 9239, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.360368378999965, "nonpadding_tokens": 6996, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.36077975299997433, "nonpadding_tokens": 6962, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003741470000022673, "prefetch": {"data_wait_seconds": 0.02183281300115425, "consumed_batches": 2327, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2877, "tokens": 209743897, "loss_tokens": 72127, "loss": 2.518863211765358, "lr": 0.0002990340339437676, "optimizer_lrs": [0.01993560226291784, 0.0002990340339437676], "gradient_norm": 0.4099177122116089, "seconds": 0.2529549490000136, "nonpadding_tokens_per_second": 285137.73019715113, "ranks": [{"compute_seconds": 0.24870787899999414, "nonpadding_tokens": 9967, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2491712839999991, "nonpadding_tokens": 4446, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2464841849999857, "nonpadding_tokens": 13424, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24739078499999323, "nonpadding_tokens": 12950, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24520209100001011, "nonpadding_tokens": 7152, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24801990100002058, "nonpadding_tokens": 3822, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24505577599995831, "nonpadding_tokens": 12193, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2504949190000616, "nonpadding_tokens": 8173, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004215229999999792, "prefetch": {"data_wait_seconds": 0.021843188001142266, "consumed_batches": 2328, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2878, "tokens": 209826861, "loss_tokens": 82964, "loss": 2.306257646388795, "lr": 0.0002990331900471642, "optimizer_lrs": [0.019935546003144282, 0.0002990331900471642], "gradient_norm": 0.3684699237346649, "seconds": 0.26267892200007736, "nonpadding_tokens_per_second": 315838.05570808443, "ranks": [{"compute_seconds": 0.26022969699999976, "nonpadding_tokens": 7117, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2583348930000966, "nonpadding_tokens": 15634, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25550614900009805, "nonpadding_tokens": 22581, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25612714199996844, "nonpadding_tokens": 6369, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25454618899993875, "nonpadding_tokens": 9804, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2571156980000069, "nonpadding_tokens": 10785, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26004582100006246, "nonpadding_tokens": 5769, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2548323259999279, "nonpadding_tokens": 4905, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039371200000459794, "prefetch": {"data_wait_seconds": 0.021852151001169773, "consumed_batches": 2329, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2879, "tokens": 209885265, "loss_tokens": 58404, "loss": 2.2843218144305184, "lr": 0.0002990325957507717, "optimizer_lrs": [0.01993550638338478, 0.0002990325957507717], "gradient_norm": 0.5154942274093628, "seconds": 0.37691054300000815, "nonpadding_tokens_per_second": 154954.54049954432, "ranks": [{"compute_seconds": 0.3731306859999677, "nonpadding_tokens": 17658, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37457031500002813, "nonpadding_tokens": 3112, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3712334040000087, "nonpadding_tokens": 5836, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3718404459999647, "nonpadding_tokens": 2472, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.37049603100001605, "nonpadding_tokens": 3973, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3721550219999017, "nonpadding_tokens": 5229, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3745044260000441, "nonpadding_tokens": 3155, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.37085594600000604, "nonpadding_tokens": 16969, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003537149999601752, "prefetch": {"data_wait_seconds": 0.021860797001139076, "consumed_batches": 2330, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2880, "tokens": 209965850, "loss_tokens": 80585, "loss": 2.4343561922193957, "lr": 0.0002990317754504345, "optimizer_lrs": [0.019935451696695635, 0.0002990317754504345], "gradient_norm": 1.4868932962417603, "seconds": 0.37492129800000384, "nonpadding_tokens_per_second": 214938.44289421823, "ranks": [{"compute_seconds": 0.36763379100000293, "nonpadding_tokens": 7167, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3697323389999383, "nonpadding_tokens": 10386, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37150333599993246, "nonpadding_tokens": 1479, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3721317640000734, "nonpadding_tokens": 13002, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3659908039999209, "nonpadding_tokens": 10467, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3684688889999279, "nonpadding_tokens": 2637, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.365972192000072, "nonpadding_tokens": 18107, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3667591349999384, "nonpadding_tokens": 17340, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041351000004397065, "prefetch": {"data_wait_seconds": 0.021870383001058258, "consumed_batches": 2331, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2881, "tokens": 210016141, "loss_tokens": 50291, "loss": 2.3873481338609293, "lr": 0.0002990312633468181, "optimizer_lrs": [0.019935417556454543, 0.0002990312633468181], "gradient_norm": 0.4187626242637634, "seconds": 0.35708030199998575, "nonpadding_tokens_per_second": 140839.4686526338, "ranks": [{"compute_seconds": 0.3498199039999008, "nonpadding_tokens": 17407, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3543778299999758, "nonpadding_tokens": 8161, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3489193300000579, "nonpadding_tokens": 6197, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3494082290000051, "nonpadding_tokens": 4610, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3476413289999982, "nonpadding_tokens": 3166, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35059100499995566, "nonpadding_tokens": 1884, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34772243899999467, "nonpadding_tokens": 2961, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.348373185000014, "nonpadding_tokens": 5905, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039199399998324225, "prefetch": {"data_wait_seconds": 0.021878545001030147, "consumed_batches": 2332, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2882, "tokens": 210070112, "loss_tokens": 53971, "loss": 2.182095332447055, "lr": 0.00029903071362039345, "optimizer_lrs": [0.019935380908026233, 0.00029903071362039345], "gradient_norm": 1.011316180229187, "seconds": 0.2733292389999633, "nonpadding_tokens_per_second": 197457.83582270629, "ranks": [{"compute_seconds": 0.2685055250000232, "nonpadding_tokens": 13035, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26731021599994165, "nonpadding_tokens": 1513, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26904980899996644, "nonpadding_tokens": 10842, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26471981600002437, "nonpadding_tokens": 7030, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2705987730000743, "nonpadding_tokens": 3075, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26769873000000644, "nonpadding_tokens": 7190, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26325038299989956, "nonpadding_tokens": 3192, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2639703190000091, "nonpadding_tokens": 8094, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004179190000286326, "prefetch": {"data_wait_seconds": 0.02188761400107353, "consumed_batches": 2333, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2883, "tokens": 210137395, "loss_tokens": 67283, "loss": 2.4805221043948693, "lr": 0.00029903002808582895, "optimizer_lrs": [0.01993533520572193, 0.00029903002808582895], "gradient_norm": 0.4257661700248718, "seconds": 0.2844950820000349, "nonpadding_tokens_per_second": 236499.6945711411, "ranks": [{"compute_seconds": 0.2763042489999634, "nonpadding_tokens": 2436, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2785595820000708, "nonpadding_tokens": 10743, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27553546100000403, "nonpadding_tokens": 3471, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.281570022999972, "nonpadding_tokens": 8284, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2744704870000305, "nonpadding_tokens": 14258, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2773247170000559, "nonpadding_tokens": 10214, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2743336560000671, "nonpadding_tokens": 4547, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28199121000000105, "nonpadding_tokens": 13330, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004367040000943234, "prefetch": {"data_wait_seconds": 0.02189729300103238, "consumed_batches": 2334, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2884, "tokens": 210204020, "loss_tokens": 66625, "loss": 2.2144922607879924, "lr": 0.00029902934901757566, "optimizer_lrs": [0.019935289934505045, 0.00029902934901757566], "gradient_norm": 0.4153374135494232, "seconds": 0.33721465199994327, "nonpadding_tokens_per_second": 197574.4517768202, "ranks": [{"compute_seconds": 0.2957122719999461, "nonpadding_tokens": 17432, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30100547999995797, "nonpadding_tokens": 12019, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29806463100010205, "nonpadding_tokens": 8207, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2989078210000571, "nonpadding_tokens": 2678, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2970764069999632, "nonpadding_tokens": 2834, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2998968000000559, "nonpadding_tokens": 10573, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2969124459999648, "nonpadding_tokens": 2312, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29799154399995587, "nonpadding_tokens": 10570, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.025950298999987353, "prefetch": {"data_wait_seconds": 0.02190553400107831, "consumed_batches": 2335, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2885, "tokens": 210283824, "loss_tokens": 79804, "loss": 2.0919612425442335, "lr": 0.0002990285353121145, "optimizer_lrs": [0.019935235687474302, 0.0002990285353121145], "gradient_norm": 0.44069570302963257, "seconds": 0.3312919239999701, "nonpadding_tokens_per_second": 240887.24843171003, "ranks": [{"compute_seconds": 0.3234904619999952, "nonpadding_tokens": 13063, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32550220699999954, "nonpadding_tokens": 8561, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32299669999997604, "nonpadding_tokens": 21697, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3273300370000243, "nonpadding_tokens": 3169, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3220558360000041, "nonpadding_tokens": 10090, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32457794000004014, "nonpadding_tokens": 2235, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3217878709999695, "nonpadding_tokens": 6687, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3286470030000146, "nonpadding_tokens": 14302, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003440340000224751, "prefetch": {"data_wait_seconds": 0.021914516001132256, "consumed_batches": 2336, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2886, "tokens": 210323389, "loss_tokens": 39565, "loss": 2.44286557721471, "lr": 0.0002990281317695829, "optimizer_lrs": [0.01993520878463886, 0.0002990281317695829], "gradient_norm": 0.4735993444919586, "seconds": 0.27472036999995453, "nonpadding_tokens_per_second": 144019.17120309116, "ranks": [{"compute_seconds": 0.270820530999913, "nonpadding_tokens": 2984, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26867909700001746, "nonpadding_tokens": 7342, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26852573100006794, "nonpadding_tokens": 1489, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26899676599998656, "nonpadding_tokens": 1808, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.267069036999942, "nonpadding_tokens": 2406, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2704154859999335, "nonpadding_tokens": 9015, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2722894539999743, "nonpadding_tokens": 13231, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26753741199991055, "nonpadding_tokens": 1290, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003918920000387516, "prefetch": {"data_wait_seconds": 0.02192359900114127, "consumed_batches": 2337, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2887, "tokens": 210380373, "loss_tokens": 56984, "loss": 2.23038100607188, "lr": 0.0002990275504155366, "optimizer_lrs": [0.019935170027702443, 0.0002990275504155366], "gradient_norm": 0.4075554609298706, "seconds": 0.25959863200000655, "nonpadding_tokens_per_second": 219508.09047406138, "ranks": [{"compute_seconds": 0.25518141099996683, "nonpadding_tokens": 8519, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25721943600001396, "nonpadding_tokens": 8217, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25376657999993313, "nonpadding_tokens": 3306, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2541879089999384, "nonpadding_tokens": 2158, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25232620199994926, "nonpadding_tokens": 8762, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25580735100004404, "nonpadding_tokens": 12751, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2525968699999339, "nonpadding_tokens": 5157, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2527915410000787, "nonpadding_tokens": 8114, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004023919999553982, "prefetch": {"data_wait_seconds": 0.021932137001115848, "consumed_batches": 2338, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2888, "tokens": 210443714, "loss_tokens": 63341, "loss": 2.4295008860769487, "lr": 0.0002990269040037534, "optimizer_lrs": [0.019935126933583562, 0.0002990269040037534], "gradient_norm": 0.4092235267162323, "seconds": 0.3055610850000221, "nonpadding_tokens_per_second": 207294.06691298867, "ranks": [{"compute_seconds": 0.30076419499994245, "nonpadding_tokens": 15066, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3025413350000008, "nonpadding_tokens": 7240, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29976698600000873, "nonpadding_tokens": 4589, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3002891419999969, "nonpadding_tokens": 9694, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3032225929999868, "nonpadding_tokens": 7278, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3002822019999485, "nonpadding_tokens": 3063, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2981864820000055, "nonpadding_tokens": 8101, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29914971499999865, "nonpadding_tokens": 8310, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003752390000499872, "prefetch": {"data_wait_seconds": 0.021940104001103578, "consumed_batches": 2339, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2889, "tokens": 210519023, "loss_tokens": 75309, "loss": 2.2840059372053805, "lr": 0.0002990261351768768, "optimizer_lrs": [0.019935075678458453, 0.0002990261351768768], "gradient_norm": 0.40204063057899475, "seconds": 0.2571650339999678, "nonpadding_tokens_per_second": 292843.07757021673, "ranks": [{"compute_seconds": 0.2532333450000124, "nonpadding_tokens": 15897, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2537373889999799, "nonpadding_tokens": 4447, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2508668039999975, "nonpadding_tokens": 8450, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25467394099996454, "nonpadding_tokens": 11476, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2500461170000108, "nonpadding_tokens": 10107, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2528349789999993, "nonpadding_tokens": 6060, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2495815349999475, "nonpadding_tokens": 5875, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2501943649999703, "nonpadding_tokens": 12997, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003315220000104091, "prefetch": {"data_wait_seconds": 0.021948238001144205, "consumed_batches": 2340, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2890, "tokens": 210605980, "loss_tokens": 86957, "loss": 2.4509840352128065, "lr": 0.0002990252470596879, "optimizer_lrs": [0.01993501647064586, 0.0002990252470596879], "gradient_norm": 0.4365565776824951, "seconds": 0.326028539000049, "nonpadding_tokens_per_second": 266715.9147070463, "ranks": [{"compute_seconds": 0.3217326440000079, "nonpadding_tokens": 18023, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32364759300003243, "nonpadding_tokens": 9897, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.320353311999952, "nonpadding_tokens": 4056, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3234922969999161, "nonpadding_tokens": 8478, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31903975000000173, "nonpadding_tokens": 5690, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32210249499996735, "nonpadding_tokens": 11650, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31926574900001015, "nonpadding_tokens": 12189, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3196556989998953, "nonpadding_tokens": 16974, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038664600003812666, "prefetch": {"data_wait_seconds": 0.02195726600116643, "consumed_batches": 2341, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2891, "tokens": 210656485, "loss_tokens": 50505, "loss": 2.6168980793980796, "lr": 0.00029902473105225443, "optimizer_lrs": [0.019934982070150296, 0.00029902473105225443], "gradient_norm": 0.5439613461494446, "seconds": 0.2719567829999505, "nonpadding_tokens_per_second": 185709.65372836165, "ranks": [{"compute_seconds": 0.2681979550000051, "nonpadding_tokens": 3381, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26937124300002324, "nonpadding_tokens": 2934, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26892431200008104, "nonpadding_tokens": 7622, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2695332050000161, "nonpadding_tokens": 10589, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26545142800000576, "nonpadding_tokens": 4822, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26813039699982255, "nonpadding_tokens": 7713, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26527236100014306, "nonpadding_tokens": 8427, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2665094199999203, "nonpadding_tokens": 5017, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003952399999889167, "prefetch": {"data_wait_seconds": 0.02196613700125738, "consumed_batches": 2342, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2892, "tokens": 210731122, "loss_tokens": 74637, "loss": 2.2939290750566075, "lr": 0.00029902396824011687, "optimizer_lrs": [0.01993493121600779, 0.00029902396824011687], "gradient_norm": 0.5056113600730896, "seconds": 0.3015540830001555, "nonpadding_tokens_per_second": 247507.84090680513, "ranks": [{"compute_seconds": 0.2988408310000068, "nonpadding_tokens": 9906, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29563768599996365, "nonpadding_tokens": 10002, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2924014489999536, "nonpadding_tokens": 2803, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29807822099996883, "nonpadding_tokens": 14493, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2912581799998861, "nonpadding_tokens": 8563, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2940966500000286, "nonpadding_tokens": 6162, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2911382830000093, "nonpadding_tokens": 9752, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2990747869998813, "nonpadding_tokens": 12956, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003865449998556869, "prefetch": {"data_wait_seconds": 0.02197466700131656, "consumed_batches": 2343, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2893, "tokens": 210769835, "loss_tokens": 38713, "loss": 1.8988897463384393, "lr": 0.00029902357246482374, "optimizer_lrs": [0.01993490483098825, 0.00029902357246482374], "gradient_norm": 0.4455162286758423, "seconds": 0.34827808399995774, "nonpadding_tokens_per_second": 111155.42946424587, "ranks": [{"compute_seconds": 0.34399152899982255, "nonpadding_tokens": 5699, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34238725199998044, "nonpadding_tokens": 2631, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3442704680001043, "nonpadding_tokens": 7287, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33885803399994074, "nonpadding_tokens": 4882, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3378283219999503, "nonpadding_tokens": 1584, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3409148850000747, "nonpadding_tokens": 13008, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33719832099995983, "nonpadding_tokens": 1382, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3454539809999915, "nonpadding_tokens": 2240, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041965500008700474, "prefetch": {"data_wait_seconds": 0.021984118001185493, "consumed_batches": 2344, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2894, "tokens": 210820926, "loss_tokens": 51091, "loss": 2.18485933799495, "lr": 0.0002990230500229478, "optimizer_lrs": [0.019934870001529853, 0.0002990230500229478], "gradient_norm": 0.5107730627059937, "seconds": 0.2772245799999382, "nonpadding_tokens_per_second": 184294.62495717872, "ranks": [{"compute_seconds": 0.2180593199998384, "nonpadding_tokens": 1893, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25971757599995726, "nonpadding_tokens": 6162, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26138840299995536, "nonpadding_tokens": 16807, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25778969099997084, "nonpadding_tokens": 3889, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2554478479999034, "nonpadding_tokens": 2956, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25820424200014713, "nonpadding_tokens": 2599, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25535295400004543, "nonpadding_tokens": 10666, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26251909699999487, "nonpadding_tokens": 6119, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004771700000674173, "prefetch": {"data_wait_seconds": 0.021995827001049406, "consumed_batches": 2345, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2895, "tokens": 210890464, "loss_tokens": 69538, "loss": 2.2475597874543416, "lr": 0.00029902233872365484, "optimizer_lrs": [0.01993482258157699, 0.00029902233872365484], "gradient_norm": 0.5466979146003723, "seconds": 0.2281818710000607, "nonpadding_tokens_per_second": 304748.1366299319, "ranks": [{"compute_seconds": 0.22131685999988804, "nonpadding_tokens": 6591, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2225598649999938, "nonpadding_tokens": 9203, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21941468299996814, "nonpadding_tokens": 12238, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22483172700003706, "nonpadding_tokens": 12976, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2184303540000201, "nonpadding_tokens": 12126, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22106626300001153, "nonpadding_tokens": 7501, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2180101660001128, "nonpadding_tokens": 3893, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2256978870000239, "nonpadding_tokens": 5010, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036411199994290655, "prefetch": {"data_wait_seconds": 0.02200404900118258, "consumed_batches": 2346, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2896, "tokens": 210963872, "loss_tokens": 73408, "loss": 2.2594852996267436, "lr": 0.00029902158755856903, "optimizer_lrs": [0.019934772503904602, 0.00029902158755856903], "gradient_norm": 0.48257824778556824, "seconds": 0.32359927300012714, "nonpadding_tokens_per_second": 226848.47008284583, "ranks": [{"compute_seconds": 0.3189594749999287, "nonpadding_tokens": 4599, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31720687899996847, "nonpadding_tokens": 3976, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31440006200000425, "nonpadding_tokens": 5860, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32061198099995636, "nonpadding_tokens": 5929, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31290662399987923, "nonpadding_tokens": 13613, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3158936499999072, "nonpadding_tokens": 10796, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32023703200002274, "nonpadding_tokens": 15497, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.320981164999921, "nonpadding_tokens": 13138, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044094700001551246, "prefetch": {"data_wait_seconds": 0.0220135210010568, "consumed_batches": 2347, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2897, "tokens": 211036085, "loss_tokens": 72213, "loss": 2.2821442728456094, "lr": 0.00029902084834117877, "optimizer_lrs": [0.01993472322274525, 0.00029902084834117877], "gradient_norm": 0.38354021310806274, "seconds": 0.2923506829999951, "nonpadding_tokens_per_second": 247008.14535125, "ranks": [{"compute_seconds": 0.2852160439999807, "nonpadding_tokens": 13530, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2876237119999132, "nonpadding_tokens": 8953, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2882016249998287, "nonpadding_tokens": 3543, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2884544040000492, "nonpadding_tokens": 9942, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.283521852000149, "nonpadding_tokens": 3755, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2860612570000285, "nonpadding_tokens": 11726, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28330293099998016, "nonpadding_tokens": 4523, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2897453240000232, "nonpadding_tokens": 16241, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000449724999953105, "prefetch": {"data_wait_seconds": 0.022025762000978943, "consumed_batches": 2348, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2898, "tokens": 211103388, "loss_tokens": 67303, "loss": 2.419433383355868, "lr": 0.00029902015913521216, "optimizer_lrs": [0.019934677275680813, 0.00029902015913521216], "gradient_norm": 0.37483716011047363, "seconds": 0.24438308300000244, "nonpadding_tokens_per_second": 275399.58647628373, "ranks": [{"compute_seconds": 0.2418384770001012, "nonpadding_tokens": 2857, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23926151999989997, "nonpadding_tokens": 11205, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23442778499997985, "nonpadding_tokens": 17290, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23460960300008082, "nonpadding_tokens": 3362, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23320964699996694, "nonpadding_tokens": 10069, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2368569089999255, "nonpadding_tokens": 12097, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2330123519998324, "nonpadding_tokens": 5694, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23447463600018636, "nonpadding_tokens": 4729, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034451100009391666, "prefetch": {"data_wait_seconds": 0.022034022000937625, "consumed_batches": 2349, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2899, "tokens": 211155011, "loss_tokens": 51623, "loss": 2.0428179420994517, "lr": 0.0002990196303341416, "optimizer_lrs": [0.019934642022276107, 0.0002990196303341416], "gradient_norm": 0.659206748008728, "seconds": 0.30576695700005985, "nonpadding_tokens_per_second": 168831.1925738591, "ranks": [{"compute_seconds": 0.3012747729999319, "nonpadding_tokens": 2400, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29951850199995533, "nonpadding_tokens": 5045, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29689869900016674, "nonpadding_tokens": 2970, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2972517829998651, "nonpadding_tokens": 14037, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3032019009999658, "nonpadding_tokens": 1751, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2986055890000898, "nonpadding_tokens": 2343, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29601793899996665, "nonpadding_tokens": 13296, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2962095909999789, "nonpadding_tokens": 9781, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042876200018326927, "prefetch": {"data_wait_seconds": 0.022044206000785493, "consumed_batches": 2350, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2900, "tokens": 211239566, "loss_tokens": 84555, "loss": 1.7628834412512566, "lr": 0.00029901876388651244, "optimizer_lrs": [0.01993458425910083, 0.00029901876388651244], "gradient_norm": 0.5616639256477356, "seconds": 0.28750438899987785, "nonpadding_tokens_per_second": 294099.8580722047, "ranks": [{"compute_seconds": 0.28472503100010726, "nonpadding_tokens": 3908, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28455554099991787, "nonpadding_tokens": 20861, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2810471159998542, "nonpadding_tokens": 10791, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2820829350000622, "nonpadding_tokens": 13276, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28037828700007594, "nonpadding_tokens": 12208, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28350439000018923, "nonpadding_tokens": 3015, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2805144060000657, "nonpadding_tokens": 9555, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2812734869999076, "nonpadding_tokens": 10941, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040874999990592187, "prefetch": {"data_wait_seconds": 0.022052768000733636, "consumed_batches": 2351, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2901, "tokens": 211304502, "loss_tokens": 64936, "loss": 2.40516407501232, "lr": 0.00029901809821881943, "optimizer_lrs": [0.01993453988125463, 0.00029901809821881943], "gradient_norm": 0.3943076431751251, "seconds": 0.2926681679998637, "nonpadding_tokens_per_second": 221875.8549786331, "ranks": [{"compute_seconds": 0.2902232769999955, "nonpadding_tokens": 12425, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28920016600000054, "nonpadding_tokens": 18064, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2860647349998544, "nonpadding_tokens": 11242, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.286540543000001, "nonpadding_tokens": 9826, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2850126989999353, "nonpadding_tokens": 4803, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2880523619999167, "nonpadding_tokens": 4437, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28473829899985503, "nonpadding_tokens": 1332, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2858418430000711, "nonpadding_tokens": 2807, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042861799988713756, "prefetch": {"data_wait_seconds": 0.022064027000581632, "consumed_batches": 2352, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2902, "tokens": 211362893, "loss_tokens": 58391, "loss": 2.6770341854909145, "lr": 0.0002990174994527698, "optimizer_lrs": [0.019934499963517986, 0.0002990174994527698], "gradient_norm": 0.40008363127708435, "seconds": 0.2664266219999263, "nonpadding_tokens_per_second": 219163.53389045392, "ranks": [{"compute_seconds": 0.2634296159999394, "nonpadding_tokens": 3661, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2632992060000561, "nonpadding_tokens": 3253, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2600888060001125, "nonpadding_tokens": 16008, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2605839369998648, "nonpadding_tokens": 19496, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26408487499998046, "nonpadding_tokens": 4697, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2617163279999204, "nonpadding_tokens": 3456, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25899667600015164, "nonpadding_tokens": 4927, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2596570430000611, "nonpadding_tokens": 2893, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003618140001435677, "prefetch": {"data_wait_seconds": 0.02207240200046101, "consumed_batches": 2353, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2903, "tokens": 211443452, "loss_tokens": 80559, "loss": 2.248020084658449, "lr": 0.0002990166730681569, "optimizer_lrs": [0.01993444487121046, 0.0002990166730681569], "gradient_norm": 0.4221086800098419, "seconds": 0.3110567670000819, "nonpadding_tokens_per_second": 258984.88168874587, "ranks": [{"compute_seconds": 0.3079389539998374, "nonpadding_tokens": 10235, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30480259499995555, "nonpadding_tokens": 15374, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3048962620000566, "nonpadding_tokens": 7406, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3053618229998847, "nonpadding_tokens": 8204, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30413837499986585, "nonpadding_tokens": 4827, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30648742799985484, "nonpadding_tokens": 9535, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30387422900003, "nonpadding_tokens": 17631, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3086097999998856, "nonpadding_tokens": 7347, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039298699994105846, "prefetch": {"data_wait_seconds": 0.022080784000493736, "consumed_batches": 2354, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2904, "tokens": 211511258, "loss_tokens": 67806, "loss": 2.383646911777719, "lr": 0.00029901597723719446, "optimizer_lrs": [0.01993439848247963, 0.00029901597723719446], "gradient_norm": 0.6126768589019775, "seconds": 0.31161044800001036, "nonpadding_tokens_per_second": 217598.60888874222, "ranks": [{"compute_seconds": 0.30643925799995486, "nonpadding_tokens": 8846, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30875773599996137, "nonpadding_tokens": 10262, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3059010140000282, "nonpadding_tokens": 6176, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3060370810001132, "nonpadding_tokens": 5201, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3090700069999457, "nonpadding_tokens": 11126, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30756936900002074, "nonpadding_tokens": 8712, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30472372500003075, "nonpadding_tokens": 10922, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30922776500005966, "nonpadding_tokens": 6561, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040986799990605505, "prefetch": {"data_wait_seconds": 0.022099146000485348, "consumed_batches": 2355, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2905, "tokens": 211565927, "loss_tokens": 54669, "loss": 2.24967817684611, "lr": 0.00029901541604069936, "optimizer_lrs": [0.019934361069379958, 0.00029901541604069936], "gradient_norm": 0.4272605776786804, "seconds": 0.21119417600016277, "nonpadding_tokens_per_second": 258856.5699840031, "ranks": [{"compute_seconds": 0.20702181000001474, "nonpadding_tokens": 5690, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20794274600007157, "nonpadding_tokens": 3286, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20796661999997923, "nonpadding_tokens": 5324, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20889332700016894, "nonpadding_tokens": 8000, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.20413447700002507, "nonpadding_tokens": 13471, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20742151300009937, "nonpadding_tokens": 10610, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20454932100005863, "nonpadding_tokens": 6181, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20487735300002896, "nonpadding_tokens": 2107, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003598179998789419, "prefetch": {"data_wait_seconds": 0.022107280000639662, "consumed_batches": 2356, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2906, "tokens": 211638180, "loss_tokens": 72253, "loss": 2.130158090321509, "lr": 0.00029901467409372063, "optimizer_lrs": [0.019934311606248045, 0.00029901467409372063], "gradient_norm": 0.4527605473995209, "seconds": 0.2754781809999258, "nonpadding_tokens_per_second": 262282.1151850841, "ranks": [{"compute_seconds": 0.2709370829998079, "nonpadding_tokens": 9030, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27324408899994523, "nonpadding_tokens": 13303, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2723973379997915, "nonpadding_tokens": 4271, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27020630699985304, "nonpadding_tokens": 10814, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2688025299999026, "nonpadding_tokens": 4536, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2706992800001444, "nonpadding_tokens": 10329, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26895351899997877, "nonpadding_tokens": 9038, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26968050100003893, "nonpadding_tokens": 10932, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034169300010944426, "prefetch": {"data_wait_seconds": 0.022115987000645987, "consumed_batches": 2357, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2907, "tokens": 211715040, "loss_tokens": 76860, "loss": 1.8954546008977362, "lr": 0.00029901388453305714, "optimizer_lrs": [0.019934258968870477, 0.00029901388453305714], "gradient_norm": 0.8296704292297363, "seconds": 0.25924591599982705, "nonpadding_tokens_per_second": 296475.2586499811, "ranks": [{"compute_seconds": 0.25604807799982154, "nonpadding_tokens": 6860, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2570353230000819, "nonpadding_tokens": 18124, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2536544050001339, "nonpadding_tokens": 11931, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2544106489999649, "nonpadding_tokens": 3978, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25321045300006517, "nonpadding_tokens": 17922, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2546120520000841, "nonpadding_tokens": 8056, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.256752681999842, "nonpadding_tokens": 3578, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25314357599995674, "nonpadding_tokens": 6411, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003320599998914986, "prefetch": {"data_wait_seconds": 0.022124561000694598, "consumed_batches": 2358, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2908, "tokens": 211797504, "loss_tokens": 82464, "loss": 2.618582995307043, "lr": 0.00029901303705376077, "optimizer_lrs": [0.01993420247025072, 0.00029901303705376077], "gradient_norm": 0.4297916889190674, "seconds": 0.38804234100007307, "nonpadding_tokens_per_second": 212512.89173101983, "ranks": [{"compute_seconds": 0.3838940890000231, "nonpadding_tokens": 17003, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3857547729999169, "nonpadding_tokens": 16262, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.38223013600008926, "nonpadding_tokens": 2377, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.38342298200018377, "nonpadding_tokens": 12691, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.38158538499988026, "nonpadding_tokens": 4947, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38483869400010917, "nonpadding_tokens": 14105, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3853944790000696, "nonpadding_tokens": 7713, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.382292740000139, "nonpadding_tokens": 7366, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004036979999000323, "prefetch": {"data_wait_seconds": 0.022134296000743348, "consumed_batches": 2359, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2909, "tokens": 211850011, "loss_tokens": 52507, "loss": 2.4943687567848096, "lr": 0.0002990124972523993, "optimizer_lrs": [0.019934166483493287, 0.0002990124972523993], "gradient_norm": 0.4107332229614258, "seconds": 0.20433644899981118, "nonpadding_tokens_per_second": 256963.45540412382, "ranks": [{"compute_seconds": 0.2012396559998706, "nonpadding_tokens": 4687, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20183879100000013, "nonpadding_tokens": 11060, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.19882907000010164, "nonpadding_tokens": 7557, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.1988732729998901, "nonpadding_tokens": 6905, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.19766018699988308, "nonpadding_tokens": 2848, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20050237000009474, "nonpadding_tokens": 3161, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20196030000010978, "nonpadding_tokens": 5421, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.19803830499995456, "nonpadding_tokens": 10868, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037234199999147677, "prefetch": {"data_wait_seconds": 0.0221429960008237, "consumed_batches": 2360, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2910, "tokens": 211907293, "loss_tokens": 57282, "loss": 2.1560406464945356, "lr": 0.00029901190819366395, "optimizer_lrs": [0.01993412721291093, 0.00029901190819366395], "gradient_norm": 0.43315982818603516, "seconds": 0.3083901349998541, "nonpadding_tokens_per_second": 185745.2411700105, "ranks": [{"compute_seconds": 0.3039519100000234, "nonpadding_tokens": 3291, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.306161111999927, "nonpadding_tokens": 2226, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3027860099998634, "nonpadding_tokens": 13567, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3043184969999402, "nonpadding_tokens": 10407, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30468900400001075, "nonpadding_tokens": 6316, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30460452800002713, "nonpadding_tokens": 10692, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3022587760001443, "nonpadding_tokens": 5543, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3028162450000309, "nonpadding_tokens": 5240, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003335890000926156, "prefetch": {"data_wait_seconds": 0.022151216000978025, "consumed_batches": 2361, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2911, "tokens": 211966783, "loss_tokens": 59490, "loss": 2.2566736531349805, "lr": 0.00029901129624374705, "optimizer_lrs": [0.019934086416249806, 0.00029901129624374705], "gradient_norm": 0.44562405347824097, "seconds": 0.3233519109999179, "nonpadding_tokens_per_second": 183979.11988840884, "ranks": [{"compute_seconds": 0.32062309499997355, "nonpadding_tokens": 5956, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3198617979999199, "nonpadding_tokens": 9747, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3166851989999486, "nonpadding_tokens": 10921, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31793018300004405, "nonpadding_tokens": 10379, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3161842090000846, "nonpadding_tokens": 4370, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3178224490000048, "nonpadding_tokens": 5373, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3158060809998915, "nonpadding_tokens": 6225, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31658529500009536, "nonpadding_tokens": 6519, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039569399996253196, "prefetch": {"data_wait_seconds": 0.02216148100087878, "consumed_batches": 2362, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2912, "tokens": 212060630, "loss_tokens": 93847, "loss": 2.3130326155870726, "lr": 0.0002990103304933416, "optimizer_lrs": [0.019934022032889444, 0.0002990103304933416], "gradient_norm": 0.372381329536438, "seconds": 0.2691921559999173, "nonpadding_tokens_per_second": 348624.57136391755, "ranks": [{"compute_seconds": 0.26453533200015045, "nonpadding_tokens": 15744, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26671430999999757, "nonpadding_tokens": 15102, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.266085842999928, "nonpadding_tokens": 20055, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2634601570000541, "nonpadding_tokens": 5959, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26240844699987065, "nonpadding_tokens": 20491, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2654850510000415, "nonpadding_tokens": 5028, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2664006459999655, "nonpadding_tokens": 7565, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2669042380000519, "nonpadding_tokens": 3903, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003589489999740181, "prefetch": {"data_wait_seconds": 0.022169773000953796, "consumed_batches": 2363, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2913, "tokens": 212097570, "loss_tokens": 36940, "loss": 2.4886676451678396, "lr": 0.0002990099502264164, "optimizer_lrs": [0.019933996681761094, 0.0002990099502264164], "gradient_norm": 0.6067884564399719, "seconds": 0.28239786699987235, "nonpadding_tokens_per_second": 130808.35344983925, "ranks": [{"compute_seconds": 0.2777856730001531, "nonpadding_tokens": 2428, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.280145075000064, "nonpadding_tokens": 1466, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27897620899989306, "nonpadding_tokens": 2850, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27750600700005634, "nonpadding_tokens": 12677, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27627000400002544, "nonpadding_tokens": 2323, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27856904600002963, "nonpadding_tokens": 3783, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2755517149998923, "nonpadding_tokens": 4088, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.276222928000152, "nonpadding_tokens": 7325, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035344299999451323, "prefetch": {"data_wait_seconds": 0.022178497001007713, "consumed_batches": 2364, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2914, "tokens": 212175994, "loss_tokens": 78424, "loss": 1.901453556946853, "lr": 0.00029900914267457255, "optimizer_lrs": [0.019933942844971506, 0.00029900914267457255], "gradient_norm": 0.5714727640151978, "seconds": 0.23453845500011994, "nonpadding_tokens_per_second": 334375.87025956955, "ranks": [{"compute_seconds": 0.2323165750001408, "nonpadding_tokens": 3925, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23173677899990253, "nonpadding_tokens": 16216, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23144196300017938, "nonpadding_tokens": 9309, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23195907500007706, "nonpadding_tokens": 3708, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2302405349998935, "nonpadding_tokens": 2697, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23230536899995968, "nonpadding_tokens": 7133, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23068415400007325, "nonpadding_tokens": 10392, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23079952499983847, "nonpadding_tokens": 25044, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034799600007318077, "prefetch": {"data_wait_seconds": 0.022187375001024634, "consumed_batches": 2365, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2915, "tokens": 212259577, "loss_tokens": 83583, "loss": 2.3616450115454097, "lr": 0.0002990082816381209, "optimizer_lrs": [0.019933885442541394, 0.0002990082816381209], "gradient_norm": 0.573792576789856, "seconds": 0.30528704099992865, "nonpadding_tokens_per_second": 273784.95898887346, "ranks": [{"compute_seconds": 0.3016808059999221, "nonpadding_tokens": 7949, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3027925050000704, "nonpadding_tokens": 6134, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2991149410001981, "nonpadding_tokens": 11565, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3022791959999722, "nonpadding_tokens": 1295, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3028880039998967, "nonpadding_tokens": 6956, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3012618059999568, "nonpadding_tokens": 3455, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29822485299996515, "nonpadding_tokens": 27732, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29931918700003735, "nonpadding_tokens": 18497, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038916600010452385, "prefetch": {"data_wait_seconds": 0.022196192001231907, "consumed_batches": 2366, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2916, "tokens": 212311795, "loss_tokens": 52218, "loss": 2.339243202774905, "lr": 0.0002990077435214457, "optimizer_lrs": [0.019933849568096382, 0.0002990077435214457], "gradient_norm": 0.4556848406791687, "seconds": 0.2065447149998363, "nonpadding_tokens_per_second": 252816.92634953835, "ranks": [{"compute_seconds": 0.2020389229999182, "nonpadding_tokens": 1713, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2043660159999945, "nonpadding_tokens": 3923, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20161248900012652, "nonpadding_tokens": 9712, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20219541299979937, "nonpadding_tokens": 11959, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.20086005399980422, "nonpadding_tokens": 5422, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20344472900001165, "nonpadding_tokens": 11325, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20063537600003656, "nonpadding_tokens": 4753, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2014053039999908, "nonpadding_tokens": 3411, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033433600015086995, "prefetch": {"data_wait_seconds": 0.022204108001190548, "consumed_batches": 2367, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2917, "tokens": 212369875, "loss_tokens": 58080, "loss": 2.438447776773416, "lr": 0.0002990071448248985, "optimizer_lrs": [0.019933809654993234, 0.0002990071448248985], "gradient_norm": 0.44412967562675476, "seconds": 0.25033195300011357, "nonpadding_tokens_per_second": 232011.93177274356, "ranks": [{"compute_seconds": 0.2460251929999231, "nonpadding_tokens": 2849, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2445788709999306, "nonpadding_tokens": 12595, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24513428600016596, "nonpadding_tokens": 8614, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24796519100004843, "nonpadding_tokens": 5969, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24406138500012275, "nonpadding_tokens": 2637, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2465727369999513, "nonpadding_tokens": 8482, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24404615299999932, "nonpadding_tokens": 14243, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24451040899998588, "nonpadding_tokens": 2691, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038801799996690534, "prefetch": {"data_wait_seconds": 0.02221329700110175, "consumed_batches": 2368, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2918, "tokens": 212428712, "loss_tokens": 58837, "loss": 2.351364521899485, "lr": 0.0002990065381416636, "optimizer_lrs": [0.01993376920944424, 0.0002990065381416636], "gradient_norm": 0.4139159023761749, "seconds": 0.24152367900001082, "nonpadding_tokens_per_second": 243607.584331296, "ranks": [{"compute_seconds": 0.23800899399998343, "nonpadding_tokens": 8779, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23921913699996367, "nonpadding_tokens": 12032, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23575927299998511, "nonpadding_tokens": 12830, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2360012750000351, "nonpadding_tokens": 4801, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2344551380001576, "nonpadding_tokens": 4545, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23634911399994962, "nonpadding_tokens": 5213, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2349870620000729, "nonpadding_tokens": 4357, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2350883439999052, "nonpadding_tokens": 6280, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003793599998971331, "prefetch": {"data_wait_seconds": 0.022235510001294756, "consumed_batches": 2369, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2919, "tokens": 212509338, "loss_tokens": 80626, "loss": 2.4642880398382654, "lr": 0.00029900570648669437, "optimizer_lrs": [0.019933713765779626, 0.00029900570648669437], "gradient_norm": 0.446515291929245, "seconds": 0.4044133550000879, "nonpadding_tokens_per_second": 199365.32511391092, "ranks": [{"compute_seconds": 0.4019297509998978, "nonpadding_tokens": 4573, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4000795570000264, "nonpadding_tokens": 7295, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3975490570001057, "nonpadding_tokens": 11582, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3986124960001689, "nonpadding_tokens": 6935, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.39697174100001575, "nonpadding_tokens": 24756, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3985610730001099, "nonpadding_tokens": 890, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.400633070999902, "nonpadding_tokens": 12111, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.39695763999998235, "nonpadding_tokens": 12484, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004746669999349251, "prefetch": {"data_wait_seconds": 0.02224351400145963, "consumed_batches": 2370, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2920, "tokens": 212546244, "loss_tokens": 36906, "loss": 2.453170724272476, "lr": 0.00029900532568665964, "optimizer_lrs": [0.019933688379110644, 0.00029900532568665964], "gradient_norm": 0.6119699478149414, "seconds": 0.21327509699995062, "nonpadding_tokens_per_second": 173044.11306871212, "ranks": [{"compute_seconds": 0.2084337889998551, "nonpadding_tokens": 2117, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20976671499988697, "nonpadding_tokens": 5859, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2084484209999573, "nonpadding_tokens": 7804, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20712655600004837, "nonpadding_tokens": 4150, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.20577876999982436, "nonpadding_tokens": 3085, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2082783850000851, "nonpadding_tokens": 3690, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20973521900009473, "nonpadding_tokens": 4970, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21081456199999593, "nonpadding_tokens": 5231, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047736099986650515, "prefetch": {"data_wait_seconds": 0.02225546900145048, "consumed_batches": 2371, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2921, "tokens": 212612393, "loss_tokens": 66149, "loss": 2.2173336898516984, "lr": 0.0002990046429724683, "optimizer_lrs": [0.019933642864831222, 0.0002990046429724683], "gradient_norm": 0.40973809361457825, "seconds": 0.31236864799984687, "nonpadding_tokens_per_second": 211765.81076098402, "ranks": [{"compute_seconds": 0.30537656499996046, "nonpadding_tokens": 15376, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30529454099996656, "nonpadding_tokens": 6969, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30425713499994345, "nonpadding_tokens": 11107, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3056280980001702, "nonpadding_tokens": 6320, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3093234850000499, "nonpadding_tokens": 4992, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3069024490000629, "nonpadding_tokens": 3786, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30365873400000964, "nonpadding_tokens": 12061, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30990078800004994, "nonpadding_tokens": 5538, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035413400019024266, "prefetch": {"data_wait_seconds": 0.022263995001324588, "consumed_batches": 2372, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2922, "tokens": 212661009, "loss_tokens": 48616, "loss": 2.393634233071417, "lr": 0.00029900414106504394, "optimizer_lrs": [0.019933609404336264, 0.00029900414106504394], "gradient_norm": 0.41286778450012207, "seconds": 0.21097135600007277, "nonpadding_tokens_per_second": 230438.86583344152, "ranks": [{"compute_seconds": 0.20721825300006458, "nonpadding_tokens": 4575, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20780855899988637, "nonpadding_tokens": 5822, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20423364299995228, "nonpadding_tokens": 4941, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.208666027000163, "nonpadding_tokens": 3779, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.20368819299983443, "nonpadding_tokens": 4836, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20656757999995534, "nonpadding_tokens": 7589, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20354674500003966, "nonpadding_tokens": 14310, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20442691500011279, "nonpadding_tokens": 2764, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003369119999661052, "prefetch": {"data_wait_seconds": 0.02227200300144716, "consumed_batches": 2373, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2923, "tokens": 212765035, "loss_tokens": 104026, "loss": 2.240481946820987, "lr": 0.0002990030666861266, "optimizer_lrs": [0.019933537779075108, 0.0002990030666861266], "gradient_norm": 0.502095103263855, "seconds": 0.3307889029999842, "nonpadding_tokens_per_second": 314478.50594917015, "ranks": [{"compute_seconds": 0.3272264429999723, "nonpadding_tokens": 24284, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32849546599982204, "nonpadding_tokens": 8130, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3258337350000602, "nonpadding_tokens": 12947, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32632270600015545, "nonpadding_tokens": 16903, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32706586799986326, "nonpadding_tokens": 17464, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3274142759998995, "nonpadding_tokens": 3502, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32535456300001897, "nonpadding_tokens": 9311, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3255842030000622, "nonpadding_tokens": 11485, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037085899998601235, "prefetch": {"data_wait_seconds": 0.022289353001454515, "consumed_batches": 2374, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2924, "tokens": 212822710, "loss_tokens": 57675, "loss": 2.6592362917208496, "lr": 0.00029900247077094776, "optimizer_lrs": [0.01993349805139652, 0.00029900247077094776], "gradient_norm": 0.47846031188964844, "seconds": 0.2914309450000019, "nonpadding_tokens_per_second": 197902.79992400817, "ranks": [{"compute_seconds": 0.28685804299993833, "nonpadding_tokens": 6695, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2887799510001514, "nonpadding_tokens": 19375, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28553338200003964, "nonpadding_tokens": 4681, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28569491000007474, "nonpadding_tokens": 9238, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2841892399999324, "nonpadding_tokens": 7694, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28701032800017856, "nonpadding_tokens": 4616, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2837567609999496, "nonpadding_tokens": 1437, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2889810579999903, "nonpadding_tokens": 3939, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004475099999581289, "prefetch": {"data_wait_seconds": 0.02229903900156671, "consumed_batches": 2375, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2925, "tokens": 212898792, "loss_tokens": 76082, "loss": 2.484938125969349, "lr": 0.000299001684397849, "optimizer_lrs": [0.01993344562652327, 0.000299001684397849], "gradient_norm": 0.40936654806137085, "seconds": 0.2154900870000347, "nonpadding_tokens_per_second": 353064.9648862398, "ranks": [{"compute_seconds": 0.21090037999988454, "nonpadding_tokens": 5069, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21305321199997707, "nonpadding_tokens": 13891, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2095493639999404, "nonpadding_tokens": 11111, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2101789490000101, "nonpadding_tokens": 4879, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21315902399987863, "nonpadding_tokens": 14025, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21165015299993684, "nonpadding_tokens": 9400, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2087554289998934, "nonpadding_tokens": 8439, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20983771700002762, "nonpadding_tokens": 9268, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034498399986659933, "prefetch": {"data_wait_seconds": 0.02230803000152548, "consumed_batches": 2376, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2926, "tokens": 212954339, "loss_tokens": 55547, "loss": 2.0471945492105785, "lr": 0.0002990011100767341, "optimizer_lrs": [0.019933407338448943, 0.0002990011100767341], "gradient_norm": 0.3928438425064087, "seconds": 0.2651145159998123, "nonpadding_tokens_per_second": 209520.7793150011, "ranks": [{"compute_seconds": 0.262339122999947, "nonpadding_tokens": 3094, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26284032900002785, "nonpadding_tokens": 8384, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26005031700015024, "nonpadding_tokens": 1471, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25983094499997605, "nonpadding_tokens": 8752, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25883135699996274, "nonpadding_tokens": 4003, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2617714559999058, "nonpadding_tokens": 6213, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2587182920001396, "nonpadding_tokens": 5952, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25912008499994954, "nonpadding_tokens": 17678, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036217399997440225, "prefetch": {"data_wait_seconds": 0.022316637001452477, "consumed_batches": 2377, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2927, "tokens": 212997488, "loss_tokens": 43149, "loss": 2.4374397074671488, "lr": 0.00029900066382960674, "optimizer_lrs": [0.01993337758864045, 0.00029900066382960674], "gradient_norm": 0.6242382526397705, "seconds": 0.22572250600001098, "nonpadding_tokens_per_second": 191159.49386100605, "ranks": [{"compute_seconds": 0.2212808589999895, "nonpadding_tokens": 2747, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22318134699980874, "nonpadding_tokens": 4471, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22018989699995473, "nonpadding_tokens": 5137, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2233822109999437, "nonpadding_tokens": 4838, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2194548349998513, "nonpadding_tokens": 4649, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22270246800007953, "nonpadding_tokens": 7737, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21935131400005048, "nonpadding_tokens": 11061, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22056438499998876, "nonpadding_tokens": 2509, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003564459998415259, "prefetch": {"data_wait_seconds": 0.022326082001612235, "consumed_batches": 2378, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2928, "tokens": 213055282, "loss_tokens": 57794, "loss": 2.1318289420182026, "lr": 0.00029900006596827935, "optimizer_lrs": [0.019933337731218623, 0.00029900006596827935], "gradient_norm": 0.42730021476745605, "seconds": 0.273828389000073, "nonpadding_tokens_per_second": 211059.19737191527, "ranks": [{"compute_seconds": 0.2693773160001456, "nonpadding_tokens": 5417, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27154470099981154, "nonpadding_tokens": 9032, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2684825169999385, "nonpadding_tokens": 4774, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27121482000006836, "nonpadding_tokens": 19277, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2674852960001317, "nonpadding_tokens": 2757, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2700048350000088, "nonpadding_tokens": 4380, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26756460000001425, "nonpadding_tokens": 5480, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26774693199990907, "nonpadding_tokens": 6677, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003784209998229926, "prefetch": {"data_wait_seconds": 0.022335332001787833, "consumed_batches": 2379, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2929, "tokens": 213100527, "loss_tokens": 45245, "loss": 2.29704041883081, "lr": 0.0002989995977982299, "optimizer_lrs": [0.019933306519881996, 0.0002989995977982299], "gradient_norm": 0.4221141040325165, "seconds": 0.22887405400001626, "nonpadding_tokens_per_second": 197685.14258936833, "ranks": [{"compute_seconds": 0.2264171609999721, "nonpadding_tokens": 5541, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2219554999999218, "nonpadding_tokens": 10831, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2222710979999647, "nonpadding_tokens": 10765, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22289686400017672, "nonpadding_tokens": 5357, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22126748699997734, "nonpadding_tokens": 2315, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22468074899984458, "nonpadding_tokens": 2943, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22094796799979122, "nonpadding_tokens": 4396, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22175027299999783, "nonpadding_tokens": 3097, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039802299988878076, "prefetch": {"data_wait_seconds": 0.02234489700181541, "consumed_batches": 2380, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2930, "tokens": 213164046, "loss_tokens": 63519, "loss": 2.3255113627418567, "lr": 0.00029899894035480506, "optimizer_lrs": [0.01993326269032034, 0.00029899894035480506], "gradient_norm": 0.4131162762641907, "seconds": 0.3153579339998487, "nonpadding_tokens_per_second": 201418.74724493368, "ranks": [{"compute_seconds": 0.31280054200010454, "nonpadding_tokens": 16217, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31278187599991725, "nonpadding_tokens": 7304, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3100713830001496, "nonpadding_tokens": 5591, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31060604300000705, "nonpadding_tokens": 4508, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31305358999998134, "nonpadding_tokens": 7324, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3117125459998533, "nonpadding_tokens": 12620, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31251781499986464, "nonpadding_tokens": 4922, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30882774700012305, "nonpadding_tokens": 5033, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003709579998485424, "prefetch": {"data_wait_seconds": 0.022353427001874593, "consumed_batches": 2381, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2931, "tokens": 213222714, "loss_tokens": 58668, "loss": 2.509056523573328, "lr": 0.0002989983329297588, "optimizer_lrs": [0.019933222195317256, 0.0002989983329297588], "gradient_norm": 0.47370463609695435, "seconds": 0.20487835900007667, "nonpadding_tokens_per_second": 286355.2806959863, "ranks": [{"compute_seconds": 0.18797514800007775, "nonpadding_tokens": 5078, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19354398500013303, "nonpadding_tokens": 4501, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.19641092899996693, "nonpadding_tokens": 19160, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.1944497780000347, "nonpadding_tokens": 6585, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.19354966099990634, "nonpadding_tokens": 3101, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19552850800005217, "nonpadding_tokens": 6751, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19304212299994106, "nonpadding_tokens": 2223, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.19350549499995395, "nonpadding_tokens": 11269, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034850099996219797, "prefetch": {"data_wait_seconds": 0.022361069001775036, "consumed_batches": 2382, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2932, "tokens": 213300641, "loss_tokens": 77927, "loss": 2.1205226526107768, "lr": 0.000298997525820903, "optimizer_lrs": [0.0199331683880602, 0.000298997525820903], "gradient_norm": 0.4393646717071533, "seconds": 0.27755873099999917, "nonpadding_tokens_per_second": 280758.5973579056, "ranks": [{"compute_seconds": 0.27519118699979117, "nonpadding_tokens": 5287, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2741996269999163, "nonpadding_tokens": 7516, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27105941500008157, "nonpadding_tokens": 3840, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27192338899999413, "nonpadding_tokens": 15211, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2697783649998655, "nonpadding_tokens": 12759, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2727630990000307, "nonpadding_tokens": 3047, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26969963300007294, "nonpadding_tokens": 16410, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27084320299991305, "nonpadding_tokens": 13857, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035564300014812034, "prefetch": {"data_wait_seconds": 0.022369752001807797, "consumed_batches": 2383, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2933, "tokens": 213342983, "loss_tokens": 42342, "loss": 2.1970269619999057, "lr": 0.0002989970871387513, "optimizer_lrs": [0.019933139142583423, 0.0002989970871387513], "gradient_norm": 0.48740553855895996, "seconds": 0.2746241249999457, "nonpadding_tokens_per_second": 154181.64736986734, "ranks": [{"compute_seconds": 0.23303666299989345, "nonpadding_tokens": 2187, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2357509860000846, "nonpadding_tokens": 5661, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23316766700008884, "nonpadding_tokens": 7991, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23408926100000826, "nonpadding_tokens": 3698, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.232064414999968, "nonpadding_tokens": 4232, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23374352199994064, "nonpadding_tokens": 1746, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2361286249999921, "nonpadding_tokens": 13792, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2326067740000326, "nonpadding_tokens": 3035, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.015982425999936822, "prefetch": {"data_wait_seconds": 0.022379898001830156, "consumed_batches": 2384, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2934, "tokens": 213409129, "loss_tokens": 66146, "loss": 2.0922273266713027, "lr": 0.0002989964016451265, "optimizer_lrs": [0.019933093443008434, 0.0002989964016451265], "gradient_norm": 0.36966177821159363, "seconds": 0.29185222699993574, "nonpadding_tokens_per_second": 226642.09445972316, "ranks": [{"compute_seconds": 0.289165278000155, "nonpadding_tokens": 6580, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2880211850001615, "nonpadding_tokens": 5127, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2882997089998298, "nonpadding_tokens": 1844, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28577945400002136, "nonpadding_tokens": 9301, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2846095590000459, "nonpadding_tokens": 6698, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28731728299999304, "nonpadding_tokens": 13915, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2845544310000605, "nonpadding_tokens": 14881, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28954857400003675, "nonpadding_tokens": 7800, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003652750001492677, "prefetch": {"data_wait_seconds": 0.022388740002000418, "consumed_batches": 2385, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2935, "tokens": 213461190, "loss_tokens": 52061, "loss": 2.330061130212635, "lr": 0.00029899586195509316, "optimizer_lrs": [0.01993305746367288, 0.00029899586195509316], "gradient_norm": 0.3982407748699188, "seconds": 0.32553488799999286, "nonpadding_tokens_per_second": 159924.4871105832, "ranks": [{"compute_seconds": 0.32325968700001795, "nonpadding_tokens": 5700, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.319098011000051, "nonpadding_tokens": 7803, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31837118499993267, "nonpadding_tokens": 3587, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3191260290000173, "nonpadding_tokens": 3806, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3174994810001408, "nonpadding_tokens": 4507, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32026981900003193, "nonpadding_tokens": 12799, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32230577199993604, "nonpadding_tokens": 5098, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3179746400001022, "nonpadding_tokens": 8761, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004187310000816069, "prefetch": {"data_wait_seconds": 0.022397445002027894, "consumed_batches": 2386, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2936, "tokens": 213527372, "loss_tokens": 66182, "loss": 2.4149719240125713, "lr": 0.0002989951756711514, "optimizer_lrs": [0.019933011711410095, 0.0002989951756711514], "gradient_norm": 0.44290968775749207, "seconds": 0.27176120000012816, "nonpadding_tokens_per_second": 243529.98146891018, "ranks": [{"compute_seconds": 0.2657937609999408, "nonpadding_tokens": 2660, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26854035000019394, "nonpadding_tokens": 8714, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2655566109999654, "nonpadding_tokens": 4064, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2662438629999997, "nonpadding_tokens": 9938, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26451174999988325, "nonpadding_tokens": 10864, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26756959199997254, "nonpadding_tokens": 15001, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26840756199999305, "nonpadding_tokens": 6321, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26942237899993415, "nonpadding_tokens": 8620, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003633410001384618, "prefetch": {"data_wait_seconds": 0.022406684002092447, "consumed_batches": 2387, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2937, "tokens": 213587959, "loss_tokens": 60587, "loss": 2.377398410550118, "lr": 0.00029899454720059903, "optimizer_lrs": [0.01993296981337327, 0.00029899454720059903], "gradient_norm": 0.4565052092075348, "seconds": 0.24976005799999257, "nonpadding_tokens_per_second": 242580.821309714, "ranks": [{"compute_seconds": 0.2459638670000004, "nonpadding_tokens": 14944, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24466438399986146, "nonpadding_tokens": 11278, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24744331600004443, "nonpadding_tokens": 6768, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24538606700002674, "nonpadding_tokens": 2186, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24441340199996375, "nonpadding_tokens": 8719, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2465766430000258, "nonpadding_tokens": 3420, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24446468399992227, "nonpadding_tokens": 1062, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2450309800001378, "nonpadding_tokens": 12210, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034822600014194904, "prefetch": {"data_wait_seconds": 0.0224143610020775, "consumed_batches": 2388, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2938, "tokens": 213678502, "loss_tokens": 90543, "loss": 2.6581310123366797, "lr": 0.0002989936076308261, "optimizer_lrs": [0.01993290717538841, 0.0002989936076308261], "gradient_norm": 0.38664114475250244, "seconds": 0.5169781119998333, "nonpadding_tokens_per_second": 175138.94282632452, "ranks": [{"compute_seconds": 0.5128499960001136, "nonpadding_tokens": 17197, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5146046249999472, "nonpadding_tokens": 13939, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.5110500579999098, "nonpadding_tokens": 14127, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.5140199099998881, "nonpadding_tokens": 11118, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.5098081789999469, "nonpadding_tokens": 3948, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5137635419998787, "nonpadding_tokens": 12956, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.5095496609999373, "nonpadding_tokens": 9153, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.51002076199984, "nonpadding_tokens": 8105, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003457239999988815, "prefetch": {"data_wait_seconds": 0.022421568002030767, "consumed_batches": 2389, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2939, "tokens": 213762479, "loss_tokens": 83977, "loss": 2.223580935851483, "lr": 0.0002989927358060506, "optimizer_lrs": [0.01993284905373671, 0.0002989927358060506], "gradient_norm": 0.43811845779418945, "seconds": 0.24560862099997394, "nonpadding_tokens_per_second": 341913.89397528075, "ranks": [{"compute_seconds": 0.24310168199986038, "nonpadding_tokens": 20142, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2431628090000686, "nonpadding_tokens": 18335, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2396363249999922, "nonpadding_tokens": 16266, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24264506400004393, "nonpadding_tokens": 3129, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23882749900008093, "nonpadding_tokens": 10694, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24174797899991063, "nonpadding_tokens": 5217, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23924193699986063, "nonpadding_tokens": 3211, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24324258899991946, "nonpadding_tokens": 6983, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004010269999525917, "prefetch": {"data_wait_seconds": 0.022431615001963223, "consumed_batches": 2390, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2940, "tokens": 213814619, "loss_tokens": 52140, "loss": 2.4464256389048713, "lr": 0.0002989921943145202, "optimizer_lrs": [0.019932812954301347, 0.0002989921943145202], "gradient_norm": 0.48647046089172363, "seconds": 0.23983571200005827, "nonpadding_tokens_per_second": 217398.81673663066, "ranks": [{"compute_seconds": 0.2005951459998414, "nonpadding_tokens": 6846, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20511845600003653, "nonpadding_tokens": 2684, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20472802200015394, "nonpadding_tokens": 1880, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20350555899995015, "nonpadding_tokens": 10007, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2012410820000241, "nonpadding_tokens": 4634, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20310358699998687, "nonpadding_tokens": 4370, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20121504300004744, "nonpadding_tokens": 11392, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20219948999988446, "nonpadding_tokens": 10327, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.018195152000089365, "prefetch": {"data_wait_seconds": 0.022440574001805658, "consumed_batches": 2391, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2941, "tokens": 213878921, "loss_tokens": 64302, "loss": 2.407386725918323, "lr": 0.0002989915263168779, "optimizer_lrs": [0.019932768421125194, 0.0002989915263168779], "gradient_norm": 0.48353254795074463, "seconds": 0.2556356989998676, "nonpadding_tokens_per_second": 251537.63833287347, "ranks": [{"compute_seconds": 0.2531473779999942, "nonpadding_tokens": 7525, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25335281799993936, "nonpadding_tokens": 1412, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25051785600021503, "nonpadding_tokens": 1842, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2508874019999894, "nonpadding_tokens": 2602, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24923754099995676, "nonpadding_tokens": 7304, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2520994200001496, "nonpadding_tokens": 8856, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24949180299995533, "nonpadding_tokens": 12472, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25006429800009755, "nonpadding_tokens": 22289, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036746700016010436, "prefetch": {"data_wait_seconds": 0.022449294001674502, "consumed_batches": 2392, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2942, "tokens": 213975740, "loss_tokens": 96819, "loss": 2.3033253416168313, "lr": 0.00029899052010233306, "optimizer_lrs": [0.01993270134015554, 0.00029899052010233306], "gradient_norm": 0.4546768367290497, "seconds": 0.36126012199997604, "nonpadding_tokens_per_second": 268003.5633714546, "ranks": [{"compute_seconds": 0.3584423690001586, "nonpadding_tokens": 12421, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35588999699984925, "nonpadding_tokens": 7874, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35886646599988126, "nonpadding_tokens": 8625, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3566062760000932, "nonpadding_tokens": 22094, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3549483339998005, "nonpadding_tokens": 18196, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3579304069999125, "nonpadding_tokens": 4142, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3554740919998949, "nonpadding_tokens": 12627, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.355876717000001, "nonpadding_tokens": 10840, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035884999988411437, "prefetch": {"data_wait_seconds": 0.02245892000151173, "consumed_batches": 2393, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2943, "tokens": 214080188, "loss_tokens": 104448, "loss": 2.39002960803462, "lr": 0.0002989894340411097, "optimizer_lrs": [0.01993262893607398, 0.0002989894340411097], "gradient_norm": 0.4690665006637573, "seconds": 0.330456459999823, "nonpadding_tokens_per_second": 316071.8964309427, "ranks": [{"compute_seconds": 0.32817952200002765, "nonpadding_tokens": 21584, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3267530800001168, "nonpadding_tokens": 17330, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32682099499993456, "nonpadding_tokens": 10575, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3276535749998857, "nonpadding_tokens": 9552, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32589708699993025, "nonpadding_tokens": 12018, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32820783299985123, "nonpadding_tokens": 10795, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3257005789998857, "nonpadding_tokens": 11497, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32632779699997627, "nonpadding_tokens": 11097, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039902100002109364, "prefetch": {"data_wait_seconds": 0.022467498001518038, "consumed_batches": 2394, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2944, "tokens": 214146224, "loss_tokens": 66036, "loss": 2.1034173121479194, "lr": 0.0002989887470917224, "optimizer_lrs": [0.01993258313944816, 0.0002989887470917224], "gradient_norm": 0.46098652482032776, "seconds": 0.34976826899992375, "nonpadding_tokens_per_second": 188799.28756491744, "ranks": [{"compute_seconds": 0.3465426039999784, "nonpadding_tokens": 22128, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3474031409998588, "nonpadding_tokens": 6051, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34676929500005826, "nonpadding_tokens": 4529, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34707177099994624, "nonpadding_tokens": 3009, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.34366133499997886, "nonpadding_tokens": 7552, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3463525130000562, "nonpadding_tokens": 5688, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34349718100020254, "nonpadding_tokens": 9233, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3439114380000774, "nonpadding_tokens": 7846, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003868209998927341, "prefetch": {"data_wait_seconds": 0.02247639300158255, "consumed_batches": 2395, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2945, "tokens": 214202675, "loss_tokens": 56451, "loss": 2.2629577255053053, "lr": 0.00029898815966738503, "optimizer_lrs": [0.01993254397782567, 0.00029898815966738503], "gradient_norm": 0.507300853729248, "seconds": 0.2767415099999653, "nonpadding_tokens_per_second": 203984.57752148237, "ranks": [{"compute_seconds": 0.27322616500009644, "nonpadding_tokens": 2017, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27444620100004613, "nonpadding_tokens": 2336, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2716345400001501, "nonpadding_tokens": 5467, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27405552800018995, "nonpadding_tokens": 3287, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2704149690000577, "nonpadding_tokens": 6197, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27331805399990117, "nonpadding_tokens": 6685, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2703852919999008, "nonpadding_tokens": 13178, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2714928609998424, "nonpadding_tokens": 17284, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037462399996002205, "prefetch": {"data_wait_seconds": 0.022485741001631254, "consumed_batches": 2396, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2946, "tokens": 214270294, "loss_tokens": 67619, "loss": 2.155478443558763, "lr": 0.00029898745580611043, "optimizer_lrs": [0.019932497053740696, 0.00029898745580611043], "gradient_norm": 0.4143269956111908, "seconds": 0.34048787299980177, "nonpadding_tokens_per_second": 198594.44450768843, "ranks": [{"compute_seconds": 0.33760120200008714, "nonpadding_tokens": 9663, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3375921819999803, "nonpadding_tokens": 5954, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33461237799997434, "nonpadding_tokens": 18570, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3354137979999905, "nonpadding_tokens": 11983, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3380168409998987, "nonpadding_tokens": 7075, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33521309000002475, "nonpadding_tokens": 5122, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3335959820001335, "nonpadding_tokens": 4094, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33650875100011035, "nonpadding_tokens": 5158, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003479740000784659, "prefetch": {"data_wait_seconds": 0.02249419000163755, "consumed_batches": 2397, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2947, "tokens": 214341819, "loss_tokens": 71525, "loss": 2.1017349702901083, "lr": 0.0002989867110210726, "optimizer_lrs": [0.01993244740140484, 0.0002989867110210726], "gradient_norm": 0.4217034876346588, "seconds": 0.2167927640000471, "nonpadding_tokens_per_second": 329923.37327266356, "ranks": [{"compute_seconds": 0.21318250800004535, "nonpadding_tokens": 5046, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21457801799988374, "nonpadding_tokens": 12565, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21131489599997622, "nonpadding_tokens": 5155, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21149847800006683, "nonpadding_tokens": 5797, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21062795700004244, "nonpadding_tokens": 12082, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21307161100003214, "nonpadding_tokens": 5119, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20971652100001847, "nonpadding_tokens": 3802, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21433815799991862, "nonpadding_tokens": 21959, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000345114000083413, "prefetch": {"data_wait_seconds": 0.02250340000155404, "consumed_batches": 2398, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2948, "tokens": 214424099, "loss_tokens": 82280, "loss": 2.6713632185828877, "lr": 0.0002989858539075455, "optimizer_lrs": [0.019932390260503037, 0.0002989858539075455], "gradient_norm": 0.4993659555912018, "seconds": 0.2777763040000991, "nonpadding_tokens_per_second": 296209.57156939723, "ranks": [{"compute_seconds": 0.18913322000003063, "nonpadding_tokens": 2612, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20699430099989513, "nonpadding_tokens": 11793, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2036417810002149, "nonpadding_tokens": 15064, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20713469199995416, "nonpadding_tokens": 25193, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.20769021399996745, "nonpadding_tokens": 6583, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20556856200005313, "nonpadding_tokens": 4797, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20234494800001812, "nonpadding_tokens": 14735, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2032636729998103, "nonpadding_tokens": 1503, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.04711883800018768, "prefetch": {"data_wait_seconds": 0.022511800001623783, "consumed_batches": 2399, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2949, "tokens": 214496165, "loss_tokens": 72066, "loss": 2.2106675651486136, "lr": 0.0002989851028970513, "optimizer_lrs": [0.019932340193136756, 0.0002989851028970513], "gradient_norm": 0.4277362823486328, "seconds": 0.24776468200002455, "nonpadding_tokens_per_second": 290864.7004014594, "ranks": [{"compute_seconds": 0.24337665999996716, "nonpadding_tokens": 13154, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24505020799983868, "nonpadding_tokens": 16964, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24159277099988685, "nonpadding_tokens": 11159, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24203974300007758, "nonpadding_tokens": 1865, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2451506080001309, "nonpadding_tokens": 7091, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2434803369999372, "nonpadding_tokens": 6226, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24064040100006423, "nonpadding_tokens": 6703, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24540500700004486, "nonpadding_tokens": 8904, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003404659998977877, "prefetch": {"data_wait_seconds": 0.022520329001466166, "consumed_batches": 2400, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2950, "tokens": 214563373, "loss_tokens": 67208, "loss": 2.3944589464795856, "lr": 0.00029898440226294634, "optimizer_lrs": [0.019932293484196423, 0.00029898440226294634], "gradient_norm": 0.38808682560920715, "seconds": 0.2987660979999873, "nonpadding_tokens_per_second": 224951.8953117728, "ranks": [{"compute_seconds": 0.29466073899993717, "nonpadding_tokens": 13625, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29649069699985375, "nonpadding_tokens": 5253, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2926246270001229, "nonpadding_tokens": 9936, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2938826690001406, "nonpadding_tokens": 7368, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29231170699995346, "nonpadding_tokens": 8258, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29361956000002465, "nonpadding_tokens": 16208, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.295957683999859, "nonpadding_tokens": 3305, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2925082939998447, "nonpadding_tokens": 3255, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037804299995514157, "prefetch": {"data_wait_seconds": 0.022531123001456876, "consumed_batches": 2401, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2951, "tokens": 214595959, "loss_tokens": 32586, "loss": 2.5611979166666665, "lr": 0.0002989840624717066, "optimizer_lrs": [0.019932270831447108, 0.0002989840624717066], "gradient_norm": 0.46075910329818726, "seconds": 0.21239052800001446, "nonpadding_tokens_per_second": 153424.92109628252, "ranks": [{"compute_seconds": 0.2084046089998992, "nonpadding_tokens": 3347, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20581834000017807, "nonpadding_tokens": 1298, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20343005999984598, "nonpadding_tokens": 4412, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20924459999991996, "nonpadding_tokens": 3014, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.20951045799984058, "nonpadding_tokens": 2411, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.205211472999963, "nonpadding_tokens": 11155, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20297189599978083, "nonpadding_tokens": 4840, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21006297200005974, "nonpadding_tokens": 2109, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038284400011434627, "prefetch": {"data_wait_seconds": 0.022539684001415594, "consumed_batches": 2402, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2952, "tokens": 214650048, "loss_tokens": 54089, "loss": 1.9853979090018303, "lr": 0.0002989834983324686, "optimizer_lrs": [0.019932233222164576, 0.0002989834983324686], "gradient_norm": 0.39981889724731445, "seconds": 0.31044926399999895, "nonpadding_tokens_per_second": 174228.1469863629, "ranks": [{"compute_seconds": 0.3080320889998802, "nonpadding_tokens": 20825, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30489676100000906, "nonpadding_tokens": 4282, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30532687400000214, "nonpadding_tokens": 5871, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3058188129998598, "nonpadding_tokens": 9553, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3038643059999231, "nonpadding_tokens": 4952, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3054023719998895, "nonpadding_tokens": 1700, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30804050999995525, "nonpadding_tokens": 4411, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3047427069998321, "nonpadding_tokens": 2495, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003433710000990686, "prefetch": {"data_wait_seconds": 0.022547497001554007, "consumed_batches": 2403, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2953, "tokens": 214700611, "loss_tokens": 50563, "loss": 2.485588677738663, "lr": 0.0002989829708277492, "optimizer_lrs": [0.01993219805518328, 0.0002989829708277492], "gradient_norm": 0.40567901730537415, "seconds": 0.2874968410001202, "nonpadding_tokens_per_second": 175873.23681229202, "ranks": [{"compute_seconds": 0.2849720740000521, "nonpadding_tokens": 7933, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2851093080000737, "nonpadding_tokens": 3825, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2838714530000743, "nonpadding_tokens": 6435, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28456506699990314, "nonpadding_tokens": 7281, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28121160599994255, "nonpadding_tokens": 10215, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2833725200000572, "nonpadding_tokens": 2065, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28488472200001524, "nonpadding_tokens": 2631, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28141387000005125, "nonpadding_tokens": 10178, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040101099989442446, "prefetch": {"data_wait_seconds": 0.02255765700142831, "consumed_batches": 2404, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2954, "tokens": 214752253, "loss_tokens": 51642, "loss": 2.4143708125169434, "lr": 0.00029898243192551596, "optimizer_lrs": [0.019932162128367733, 0.00029898243192551596], "gradient_norm": 0.4578627347946167, "seconds": 0.2917916789999708, "nonpadding_tokens_per_second": 176982.42861821005, "ranks": [{"compute_seconds": 0.2894115359999887, "nonpadding_tokens": 4006, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28858719100003327, "nonpadding_tokens": 6942, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28902995799990094, "nonpadding_tokens": 10529, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28628124300007585, "nonpadding_tokens": 10524, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28439635300014743, "nonpadding_tokens": 4410, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2872569830001339, "nonpadding_tokens": 4324, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28458624600011717, "nonpadding_tokens": 3664, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2852926740001749, "nonpadding_tokens": 7243, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004037940000216622, "prefetch": {"data_wait_seconds": 0.022567347001270832, "consumed_batches": 2405, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2955, "tokens": 214794926, "loss_tokens": 42673, "loss": 2.4312145560424625, "lr": 0.0002989819865106192, "optimizer_lrs": [0.01993213243404128, 0.0002989819865106192], "gradient_norm": 0.4999309778213501, "seconds": 0.6854993219999415, "nonpadding_tokens_per_second": 62250.97331314887, "ranks": [{"compute_seconds": 0.6827505419998943, "nonpadding_tokens": 3629, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.6830523870000889, "nonpadding_tokens": 2389, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.6801316800001587, "nonpadding_tokens": 5581, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.680374545000177, "nonpadding_tokens": 9718, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.6787110990001111, "nonpadding_tokens": 7800, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.6822295720000966, "nonpadding_tokens": 4993, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.6829275059999418, "nonpadding_tokens": 2517, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.6793184179998661, "nonpadding_tokens": 6046, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035066600003119675, "prefetch": {"data_wait_seconds": 0.02257582000125069, "consumed_batches": 2406, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2956, "tokens": 214859991, "loss_tokens": 65065, "loss": 2.606522563974487, "lr": 0.00029898130718422584, "optimizer_lrs": [0.019932087145615056, 0.00029898130718422584], "gradient_norm": 0.46596333384513855, "seconds": 0.28042690300003414, "nonpadding_tokens_per_second": 232021.2479755984, "ranks": [{"compute_seconds": 0.27585757700012437, "nonpadding_tokens": 1886, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2769529180000063, "nonpadding_tokens": 6353, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27399811899999804, "nonpadding_tokens": 13366, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2780922039999041, "nonpadding_tokens": 14539, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27281087400001525, "nonpadding_tokens": 2850, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27600649099986185, "nonpadding_tokens": 8160, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.272701841000071, "nonpadding_tokens": 8642, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27347644099995705, "nonpadding_tokens": 9269, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038071799986028054, "prefetch": {"data_wait_seconds": 0.02258378600113531, "consumed_batches": 2407, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2957, "tokens": 214921050, "loss_tokens": 61059, "loss": 2.0684602955747717, "lr": 0.0002989806694781085, "optimizer_lrs": [0.0199320446318739, 0.0002989806694781085], "gradient_norm": 0.40060150623321533, "seconds": 0.2618196070000067, "nonpadding_tokens_per_second": 233210.1888763374, "ranks": [{"compute_seconds": 0.2581839370000125, "nonpadding_tokens": 7406, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2595320359998823, "nonpadding_tokens": 4164, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2568341999999575, "nonpadding_tokens": 7885, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2592802639999263, "nonpadding_tokens": 8890, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25555738699995345, "nonpadding_tokens": 2929, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2580194649999612, "nonpadding_tokens": 4604, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2558395380001457, "nonpadding_tokens": 12439, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2562820620000821, "nonpadding_tokens": 12742, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040069199985737214, "prefetch": {"data_wait_seconds": 0.02259310200129505, "consumed_batches": 2408, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2958, "tokens": 214986002, "loss_tokens": 64952, "loss": 2.306266069559059, "lr": 0.000298979990894922, "optimizer_lrs": [0.019931999392994802, 0.000298979990894922], "gradient_norm": 0.37500062584877014, "seconds": 0.37525427800005673, "nonpadding_tokens_per_second": 173087.96676793697, "ranks": [{"compute_seconds": 0.3700026470000921, "nonpadding_tokens": 11070, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.37166118100003587, "nonpadding_tokens": 6143, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37125540000010915, "nonpadding_tokens": 8099, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.36940986299987344, "nonpadding_tokens": 4887, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3682067449999522, "nonpadding_tokens": 16997, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3709753130001445, "nonpadding_tokens": 6490, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3717644799999107, "nonpadding_tokens": 4925, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3728282339998259, "nonpadding_tokens": 6341, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003817350000190345, "prefetch": {"data_wait_seconds": 0.02260262400109525, "consumed_batches": 2409, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2959, "tokens": 215068926, "loss_tokens": 82924, "loss": 2.64655814360137, "lr": 0.00029897912422313904, "optimizer_lrs": [0.019931941614875937, 0.00029897912422313904], "gradient_norm": 1.770983338356018, "seconds": 0.35079170299991347, "nonpadding_tokens_per_second": 236390.9958270035, "ranks": [{"compute_seconds": 0.34823574100005317, "nonpadding_tokens": 15530, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34503437300008954, "nonpadding_tokens": 22691, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34598327399999107, "nonpadding_tokens": 3878, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34613540200007265, "nonpadding_tokens": 17543, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3445156880000013, "nonpadding_tokens": 7581, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3460684819999642, "nonpadding_tokens": 5527, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34472545599987825, "nonpadding_tokens": 7768, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3449761260001196, "nonpadding_tokens": 2406, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040383600003224274, "prefetch": {"data_wait_seconds": 0.022612490001122865, "consumed_batches": 2410, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2960, "tokens": 215133797, "loss_tokens": 64871, "loss": 1.9677092325538377, "lr": 0.0002989784459748575, "optimizer_lrs": [0.019931896398323833, 0.0002989784459748575], "gradient_norm": 0.5183537602424622, "seconds": 0.2462195809998775, "nonpadding_tokens_per_second": 263468.0789259903, "ranks": [{"compute_seconds": 0.24125742499995795, "nonpadding_tokens": 4880, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2438436750001074, "nonpadding_tokens": 3973, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23979202800001076, "nonpadding_tokens": 7267, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24063609599988922, "nonpadding_tokens": 11413, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2433405300000686, "nonpadding_tokens": 7687, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24040538899998865, "nonpadding_tokens": 9021, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23949805100005506, "nonpadding_tokens": 9803, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2395929750000505, "nonpadding_tokens": 10827, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003877259998716909, "prefetch": {"data_wait_seconds": 0.022621410001150366, "consumed_batches": 2411, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2961, "tokens": 215192486, "loss_tokens": 58689, "loss": 2.084488122774285, "lr": 0.00029897783216815846, "optimizer_lrs": [0.019931855477877233, 0.00029897783216815846], "gradient_norm": 0.36380118131637573, "seconds": 0.24344750099999146, "nonpadding_tokens_per_second": 241074.5633408743, "ranks": [{"compute_seconds": 0.24083690400016167, "nonpadding_tokens": 9756, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23699550000014824, "nonpadding_tokens": 9543, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23751029899995046, "nonpadding_tokens": 5472, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23801489200013748, "nonpadding_tokens": 4183, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24005310500001542, "nonpadding_tokens": 11972, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23923209500003395, "nonpadding_tokens": 5158, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24006531100008033, "nonpadding_tokens": 5455, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24044236999998247, "nonpadding_tokens": 7150, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003865850001147919, "prefetch": {"data_wait_seconds": 0.02262953600116191, "consumed_batches": 2412, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2962, "tokens": 215289474, "loss_tokens": 96988, "loss": 2.5760324408689734, "lr": 0.0002989768174037641, "optimizer_lrs": [0.019931787826917607, 0.0002989768174037641], "gradient_norm": 0.5016404390335083, "seconds": 0.2809542289999172, "nonpadding_tokens_per_second": 345209.2547075652, "ranks": [{"compute_seconds": 0.2786317360000794, "nonpadding_tokens": 7566, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2759620489998724, "nonpadding_tokens": 11908, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2760341289999815, "nonpadding_tokens": 12533, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2767759490000117, "nonpadding_tokens": 11602, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27564189300005637, "nonpadding_tokens": 20130, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2767831720000231, "nonpadding_tokens": 9562, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2758994130001611, "nonpadding_tokens": 11009, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2760774030000448, "nonpadding_tokens": 12678, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040803000001687906, "prefetch": {"data_wait_seconds": 0.022637798001326814, "consumed_batches": 2413, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2963, "tokens": 215370586, "loss_tokens": 81112, "loss": 2.4261979192351317, "lr": 0.0002989759683613975, "optimizer_lrs": [0.01993173122409317, 0.0002989759683613975], "gradient_norm": 0.4301072359085083, "seconds": 0.4292005889999473, "nonpadding_tokens_per_second": 188983.89722389213, "ranks": [{"compute_seconds": 0.42647826799998256, "nonpadding_tokens": 17699, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4234399899999062, "nonpadding_tokens": 3548, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.42351066299988815, "nonpadding_tokens": 13250, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4241216729999451, "nonpadding_tokens": 10599, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.4227469079999082, "nonpadding_tokens": 7432, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4233933389998583, "nonpadding_tokens": 5415, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.42248380399996677, "nonpadding_tokens": 12101, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4266172089999145, "nonpadding_tokens": 11068, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003795790000822308, "prefetch": {"data_wait_seconds": 0.02264569000131189, "consumed_batches": 2414, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2964, "tokens": 215414915, "loss_tokens": 44329, "loss": 2.2876985861399985, "lr": 0.00029897550419797996, "optimizer_lrs": [0.019931700279865332, 0.00029897550419797996], "gradient_norm": 0.4601064622402191, "seconds": 0.3286836559998392, "nonpadding_tokens_per_second": 134868.28198120595, "ranks": [{"compute_seconds": 0.32608351799990487, "nonpadding_tokens": 6616, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32610587099998156, "nonpadding_tokens": 7698, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32291203700015103, "nonpadding_tokens": 4294, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3239418030000252, "nonpadding_tokens": 7793, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3259767810000085, "nonpadding_tokens": 3690, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32535229500012974, "nonpadding_tokens": 7252, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3215779370000291, "nonpadding_tokens": 2585, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32633611699998255, "nonpadding_tokens": 4401, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003516409999519965, "prefetch": {"data_wait_seconds": 0.022653936001191255, "consumed_batches": 2415, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2965, "tokens": 215464298, "loss_tokens": 49383, "loss": 2.9366033098434685, "lr": 0.0002989749869913904, "optimizer_lrs": [0.019931665799426027, 0.0002989749869913904], "gradient_norm": 0.5444539189338684, "seconds": 0.2736572740000156, "nonpadding_tokens_per_second": 180455.64540702538, "ranks": [{"compute_seconds": 0.2703885870000704, "nonpadding_tokens": 3019, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2709323870001299, "nonpadding_tokens": 9293, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26743923799995173, "nonpadding_tokens": 3391, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26873903699993207, "nonpadding_tokens": 3311, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2667745929998091, "nonpadding_tokens": 1670, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.269147736999912, "nonpadding_tokens": 16492, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26696591999984776, "nonpadding_tokens": 4762, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2713400779998665, "nonpadding_tokens": 7445, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036743100008607144, "prefetch": {"data_wait_seconds": 0.022662675001313914, "consumed_batches": 2416, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2966, "tokens": 215526287, "loss_tokens": 61989, "loss": 2.3566870231008727, "lr": 0.0002989743375734125, "optimizer_lrs": [0.019931622504894168, 0.0002989743375734125], "gradient_norm": 0.5824240446090698, "seconds": 0.2918015350001042, "nonpadding_tokens_per_second": 212435.48290442635, "ranks": [{"compute_seconds": 0.2887907960000575, "nonpadding_tokens": 4244, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28887218199997733, "nonpadding_tokens": 3457, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2885255060000418, "nonpadding_tokens": 4069, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2864736360002098, "nonpadding_tokens": 14889, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28949636100014686, "nonpadding_tokens": 7296, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2866044929999134, "nonpadding_tokens": 5239, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28511608899998464, "nonpadding_tokens": 9295, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28630761699992036, "nonpadding_tokens": 13500, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003968369999256538, "prefetch": {"data_wait_seconds": 0.02267121800146299, "consumed_batches": 2417, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2967, "tokens": 215629471, "loss_tokens": 103184, "loss": 2.3230366990037217, "lr": 0.000298973256128174, "optimizer_lrs": [0.019931550408544937, 0.000298973256128174], "gradient_norm": 0.4316020607948303, "seconds": 0.24229401699994924, "nonpadding_tokens_per_second": 425862.764906909, "ranks": [{"compute_seconds": 0.2237009489999764, "nonpadding_tokens": 2267, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22655496299989863, "nonpadding_tokens": 2884, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2252731340001901, "nonpadding_tokens": 7243, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22606937599994126, "nonpadding_tokens": 18112, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22263417699991805, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22495621699999901, "nonpadding_tokens": 7462, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22228206100021453, "nonpadding_tokens": 17904, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22349085399991964, "nonpadding_tokens": 14544, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0007786440000927541, "prefetch": {"data_wait_seconds": 0.0226816870014801, "consumed_batches": 2418, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2968, "tokens": 215694353, "loss_tokens": 64882, "loss": 2.262427608967048, "lr": 0.00029897257582575535, "optimizer_lrs": [0.01993150505505036, 0.00029897257582575535], "gradient_norm": 0.397770494222641, "seconds": 0.29835812000010264, "nonpadding_tokens_per_second": 217463.49655232337, "ranks": [{"compute_seconds": 0.2953241899999739, "nonpadding_tokens": 9763, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2917840719999276, "nonpadding_tokens": 2632, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29324890199995934, "nonpadding_tokens": 4020, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2944736140000259, "nonpadding_tokens": 1649, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2919605990000491, "nonpadding_tokens": 14551, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2946052739998777, "nonpadding_tokens": 6712, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29605014499998106, "nonpadding_tokens": 17546, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2956998800000292, "nonpadding_tokens": 8009, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003651730000910902, "prefetch": {"data_wait_seconds": 0.022691044001433625, "consumed_batches": 2419, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2969, "tokens": 215774056, "loss_tokens": 79703, "loss": 2.432595463784299, "lr": 0.000298971739814603, "optimizer_lrs": [0.019931449320973536, 0.000298971739814603], "gradient_norm": 0.3893274962902069, "seconds": 0.2931459220001216, "nonpadding_tokens_per_second": 271888.4828968112, "ranks": [{"compute_seconds": 0.2889176989999669, "nonpadding_tokens": 10577, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2905175529999724, "nonpadding_tokens": 10401, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28762493699991865, "nonpadding_tokens": 16569, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28976858300006825, "nonpadding_tokens": 17049, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28700920899996163, "nonpadding_tokens": 6417, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.289587198999925, "nonpadding_tokens": 6558, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29024286799995025, "nonpadding_tokens": 6831, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29081090399995446, "nonpadding_tokens": 5301, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003634259999216738, "prefetch": {"data_wait_seconds": 0.02270004000156689, "consumed_batches": 2420, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2970, "tokens": 215849958, "loss_tokens": 75902, "loss": 2.577676642249216, "lr": 0.00029897094335759605, "optimizer_lrs": [0.01993139622383974, 0.00029897094335759605], "gradient_norm": 0.408882200717926, "seconds": 0.312207029000092, "nonpadding_tokens_per_second": 243114.32142668904, "ranks": [{"compute_seconds": 0.30869574200005445, "nonpadding_tokens": 4793, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3095630100001472, "nonpadding_tokens": 24833, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3065186520000225, "nonpadding_tokens": 5601, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3072791160000179, "nonpadding_tokens": 3336, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30944530100009615, "nonpadding_tokens": 7711, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30778812899984587, "nonpadding_tokens": 6085, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30545294499984266, "nonpadding_tokens": 11311, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3098239440000725, "nonpadding_tokens": 12232, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003826049999133829, "prefetch": {"data_wait_seconds": 0.022707908001578403, "consumed_batches": 2421, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2971, "tokens": 215934178, "loss_tokens": 84220, "loss": 2.358689666943719, "lr": 0.0002989700592584261, "optimizer_lrs": [0.019931337283895074, 0.0002989700592584261], "gradient_norm": 0.39467355608940125, "seconds": 0.4433731809999699, "nonpadding_tokens_per_second": 189952.8514784157, "ranks": [{"compute_seconds": 0.43870516900005896, "nonpadding_tokens": 25389, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4403294440000991, "nonpadding_tokens": 10367, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.44030693600007, "nonpadding_tokens": 10136, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.44071123899993836, "nonpadding_tokens": 7751, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.4360930709999593, "nonpadding_tokens": 12527, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4385212590000265, "nonpadding_tokens": 4628, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.44099684399998296, "nonpadding_tokens": 5629, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.437506699000096, "nonpadding_tokens": 7793, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041838900006041513, "prefetch": {"data_wait_seconds": 0.02271894700147925, "consumed_batches": 2422, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2972, "tokens": 216017418, "loss_tokens": 83240, "loss": 2.385150243272465, "lr": 0.00029896918507521465, "optimizer_lrs": [0.01993127900501431, 0.00029896918507521465], "gradient_norm": 0.46045610308647156, "seconds": 0.23010985600012646, "nonpadding_tokens_per_second": 361740.26374582696, "ranks": [{"compute_seconds": 0.22559572199998001, "nonpadding_tokens": 2828, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22391431999994893, "nonpadding_tokens": 11439, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2251822240000365, "nonpadding_tokens": 18238, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22472511400019357, "nonpadding_tokens": 13841, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22780549499998415, "nonpadding_tokens": 6950, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22624184200003583, "nonpadding_tokens": 5301, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22334513900000275, "nonpadding_tokens": 14706, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22444143400002758, "nonpadding_tokens": 9937, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033712300000843243, "prefetch": {"data_wait_seconds": 0.022728128001475056, "consumed_batches": 2423, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2973, "tokens": 216059222, "loss_tokens": 41804, "loss": 1.966626810532485, "lr": 0.0002989687459119013, "optimizer_lrs": [0.019931249727460087, 0.0002989687459119013], "gradient_norm": 0.48298710584640503, "seconds": 0.2302614730001551, "nonpadding_tokens_per_second": 181550.1284488519, "ranks": [{"compute_seconds": 0.22654388299997663, "nonpadding_tokens": 1294, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22657918300001256, "nonpadding_tokens": 2905, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2241068850000829, "nonpadding_tokens": 5508, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22480407599982755, "nonpadding_tokens": 4818, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22300581600006808, "nonpadding_tokens": 6658, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22557831099993564, "nonpadding_tokens": 4214, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22294764600019334, "nonpadding_tokens": 12915, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2278792520000934, "nonpadding_tokens": 3492, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003876539999509987, "prefetch": {"data_wait_seconds": 0.022737131001534294, "consumed_batches": 2424, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2974, "tokens": 216149572, "loss_tokens": 90350, "loss": 2.1060649557277253, "lr": 0.0002989677964402923, "optimizer_lrs": [0.019931186429352824, 0.0002989677964402923], "gradient_norm": 0.3805512487888336, "seconds": 0.2696202029999313, "nonpadding_tokens_per_second": 335101.00131488673, "ranks": [{"compute_seconds": 0.2367894599999545, "nonpadding_tokens": 6573, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23850999400019646, "nonpadding_tokens": 3695, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23568786299983913, "nonpadding_tokens": 12812, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2364843570001085, "nonpadding_tokens": 12734, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23518147200002204, "nonpadding_tokens": 16410, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23901945099987643, "nonpadding_tokens": 22537, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24146957999982988, "nonpadding_tokens": 8462, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23492885600012414, "nonpadding_tokens": 7127, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00149885600012567, "prefetch": {"data_wait_seconds": 0.022746729001440258, "consumed_batches": 2425, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2975, "tokens": 216246229, "loss_tokens": 96657, "loss": 2.5121371512151214, "lr": 0.00029896678020773534, "optimizer_lrs": [0.01993111868051569, 0.00029896678020773534], "gradient_norm": 0.42063745856285095, "seconds": 0.2826659810000365, "nonpadding_tokens_per_second": 341947.7634274905, "ranks": [{"compute_seconds": 0.27950127899998733, "nonpadding_tokens": 10568, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2789097299998957, "nonpadding_tokens": 7672, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2791615629998887, "nonpadding_tokens": 4884, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27762199100016005, "nonpadding_tokens": 20759, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27578681599993615, "nonpadding_tokens": 12151, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2775831050000761, "nonpadding_tokens": 6253, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2802475380001397, "nonpadding_tokens": 10898, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27629337899998063, "nonpadding_tokens": 23472, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000384670999892478, "prefetch": {"data_wait_seconds": 0.022756248001499557, "consumed_batches": 2426, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2976, "tokens": 216306997, "loss_tokens": 60768, "loss": 2.188809282846235, "lr": 0.0002989661410500072, "optimizer_lrs": [0.01993107607000048, 0.0002989661410500072], "gradient_norm": 0.4080727696418762, "seconds": 0.3322705999999016, "nonpadding_tokens_per_second": 182887.0805903923, "ranks": [{"compute_seconds": 0.3289147180000782, "nonpadding_tokens": 11264, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32933493399991676, "nonpadding_tokens": 18702, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3262235419999797, "nonpadding_tokens": 5683, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3264847500001906, "nonpadding_tokens": 2780, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3249878910000916, "nonpadding_tokens": 2515, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32796922399984396, "nonpadding_tokens": 6706, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32484297700011666, "nonpadding_tokens": 8737, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3298488590000943, "nonpadding_tokens": 4381, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003835099998923397, "prefetch": {"data_wait_seconds": 0.022765868001400236, "consumed_batches": 2427, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2977, "tokens": 216361108, "loss_tokens": 54111, "loss": 2.2347410011365527, "lr": 0.0002989655717448908, "optimizer_lrs": [0.019931038116326055, 0.0002989655717448908], "gradient_norm": 0.4472503960132599, "seconds": 0.2365160399999695, "nonpadding_tokens_per_second": 228783.6376763579, "ranks": [{"compute_seconds": 0.23172695000016574, "nonpadding_tokens": 2251, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23003623699992204, "nonpadding_tokens": 3044, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22989608300008513, "nonpadding_tokens": 3264, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23312992399996801, "nonpadding_tokens": 5057, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22951817600005597, "nonpadding_tokens": 6310, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23233261499990476, "nonpadding_tokens": 14347, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22909413000002132, "nonpadding_tokens": 12165, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23399806800011902, "nonpadding_tokens": 7673, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003640340000856668, "prefetch": {"data_wait_seconds": 0.02277492500134315, "consumed_batches": 2428, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2978, "tokens": 216411570, "loss_tokens": 50462, "loss": 2.1451886134616145, "lr": 0.00029896504069045924, "optimizer_lrs": [0.019931002712697283, 0.00029896504069045924], "gradient_norm": 0.502528190612793, "seconds": 0.28227440599994225, "nonpadding_tokens_per_second": 178769.30719680738, "ranks": [{"compute_seconds": 0.2768999580000582, "nonpadding_tokens": 2125, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2755840659999649, "nonpadding_tokens": 10795, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2783208950002063, "nonpadding_tokens": 12513, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.279364706999786, "nonpadding_tokens": 4122, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2795863300000292, "nonpadding_tokens": 1735, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27500944200005506, "nonpadding_tokens": 4723, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2719122660000721, "nonpadding_tokens": 9466, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2798218259999885, "nonpadding_tokens": 4983, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003722070000549138, "prefetch": {"data_wait_seconds": 0.02278303200132825, "consumed_batches": 2429, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2979, "tokens": 216456790, "loss_tokens": 45220, "loss": 2.2125024187306503, "lr": 0.0002989645646867047, "optimizer_lrs": [0.019930970979113647, 0.0002989645646867047], "gradient_norm": 0.46136364340782166, "seconds": 0.23014932799992494, "nonpadding_tokens_per_second": 196481.12985154902, "ranks": [{"compute_seconds": 0.22618092300012904, "nonpadding_tokens": 7119, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2238396879999982, "nonpadding_tokens": 3985, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2242900850001206, "nonpadding_tokens": 3985, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22678250800004207, "nonpadding_tokens": 3464, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22773644600010812, "nonpadding_tokens": 9038, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2258007529999304, "nonpadding_tokens": 10578, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2226242540000385, "nonpadding_tokens": 2124, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22434117200009496, "nonpadding_tokens": 4927, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039813199987293046, "prefetch": {"data_wait_seconds": 0.022793292001509258, "consumed_batches": 2430, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2980, "tokens": 216512936, "loss_tokens": 56146, "loss": 2.32320590514017, "lr": 0.0002989639735198075, "optimizer_lrs": [0.019930931567987167, 0.0002989639735198075], "gradient_norm": 0.38456693291664124, "seconds": 0.31491642199989656, "nonpadding_tokens_per_second": 178288.57461113427, "ranks": [{"compute_seconds": 0.31183873800000583, "nonpadding_tokens": 3081, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30881091700007346, "nonpadding_tokens": 7820, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3086803260000579, "nonpadding_tokens": 10447, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30951646099993013, "nonpadding_tokens": 5490, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3124155860000428, "nonpadding_tokens": 5911, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3110582910001085, "nonpadding_tokens": 5026, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3076685250000537, "nonpadding_tokens": 5908, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3085901259998991, "nonpadding_tokens": 12463, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004207209999549377, "prefetch": {"data_wait_seconds": 0.022803142001706078, "consumed_batches": 2431, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2981, "tokens": 216559228, "loss_tokens": 46292, "loss": 2.158587235645468, "lr": 0.00029896348598026457, "optimizer_lrs": [0.019930899065350973, 0.00029896348598026457], "gradient_norm": 0.37908676266670227, "seconds": 0.2590676510001231, "nonpadding_tokens_per_second": 178686.91757265368, "ranks": [{"compute_seconds": 0.21186586599992552, "nonpadding_tokens": 7704, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21520767300012267, "nonpadding_tokens": 2217, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21463195599994833, "nonpadding_tokens": 10958, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21871409599998515, "nonpadding_tokens": 5203, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21428761299989674, "nonpadding_tokens": 7652, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21598934200005715, "nonpadding_tokens": 6837, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21834817100011605, "nonpadding_tokens": 3304, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21426035399986176, "nonpadding_tokens": 2417, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.028548920000048383, "prefetch": {"data_wait_seconds": 0.022811376001712347, "consumed_batches": 2432, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2982, "tokens": 216626862, "loss_tokens": 67634, "loss": 2.197257481444244, "lr": 0.00029896277346498146, "optimizer_lrs": [0.019930851564332098, 0.00029896277346498146], "gradient_norm": 0.38872435688972473, "seconds": 0.31872415599991655, "nonpadding_tokens_per_second": 212202.3032355844, "ranks": [{"compute_seconds": 0.3143366269998751, "nonpadding_tokens": 9557, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31242214999997486, "nonpadding_tokens": 10817, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31274215799999183, "nonpadding_tokens": 20145, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3138865729999907, "nonpadding_tokens": 2551, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31204143500008286, "nonpadding_tokens": 3705, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31470275400010905, "nonpadding_tokens": 10473, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31604286400011006, "nonpadding_tokens": 5289, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3164246229998753, "nonpadding_tokens": 5097, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037450700006047555, "prefetch": {"data_wait_seconds": 0.02281957800160228, "consumed_batches": 2433, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2983, "tokens": 216674859, "loss_tokens": 47997, "loss": 2.606767577921537, "lr": 0.0002989622676749679, "optimizer_lrs": [0.019930817844997863, 0.0002989622676749679], "gradient_norm": 0.4420977532863617, "seconds": 0.2940426320001279, "nonpadding_tokens_per_second": 163231.43237263337, "ranks": [{"compute_seconds": 0.2897371159999693, "nonpadding_tokens": 4086, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2878588450000734, "nonpadding_tokens": 4469, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28784469799984436, "nonpadding_tokens": 6939, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28853483500006405, "nonpadding_tokens": 12778, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2865200230000937, "nonpadding_tokens": 3324, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.288439697000058, "nonpadding_tokens": 8546, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29086677500004043, "nonpadding_tokens": 4930, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2916246569998293, "nonpadding_tokens": 2925, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003789990000768739, "prefetch": {"data_wait_seconds": 0.022828551001524033, "consumed_batches": 2434, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2984, "tokens": 216747585, "loss_tokens": 72726, "loss": 2.336493095660424, "lr": 0.00029896150105794393, "optimizer_lrs": [0.019930766737196265, 0.00029896150105794393], "gradient_norm": 0.4418061673641205, "seconds": 0.27878269799998634, "nonpadding_tokens_per_second": 260869.8478124477, "ranks": [{"compute_seconds": 0.2754273400000784, "nonpadding_tokens": 5706, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2764310200000182, "nonpadding_tokens": 19146, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2732293359999858, "nonpadding_tokens": 7563, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2745941749999474, "nonpadding_tokens": 5902, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.272159911000017, "nonpadding_tokens": 2918, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27509299600001214, "nonpadding_tokens": 16167, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27633799499994893, "nonpadding_tokens": 10354, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27332926200006114, "nonpadding_tokens": 4970, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039100999993024743, "prefetch": {"data_wait_seconds": 0.022837035001614936, "consumed_batches": 2435, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2985, "tokens": 216808550, "loss_tokens": 60965, "loss": 2.098796825022554, "lr": 0.0002989608581983461, "optimizer_lrs": [0.019930723879889742, 0.0002989608581983461], "gradient_norm": 0.4518062174320221, "seconds": 0.38178823500015824, "nonpadding_tokens_per_second": 159682.76235640087, "ranks": [{"compute_seconds": 0.37959898500002964, "nonpadding_tokens": 10836, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3760604170001898, "nonpadding_tokens": 7337, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37918617299988, "nonpadding_tokens": 5201, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3773396069998398, "nonpadding_tokens": 4075, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3759838840001066, "nonpadding_tokens": 15290, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3770617969998966, "nonpadding_tokens": 12219, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3759757800000898, "nonpadding_tokens": 3398, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3759865949998584, "nonpadding_tokens": 2609, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003564900000583293, "prefetch": {"data_wait_seconds": 0.022845678001615966, "consumed_batches": 2436, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2986, "tokens": 216879226, "loss_tokens": 70676, "loss": 2.2930041226866265, "lr": 0.00029896011269085396, "optimizer_lrs": [0.019930674179390265, 0.00029896011269085396], "gradient_norm": 0.3728239834308624, "seconds": 0.34749653100016076, "nonpadding_tokens_per_second": 203386.2030121023, "ranks": [{"compute_seconds": 0.3428193580000425, "nonpadding_tokens": 15235, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3451466670001082, "nonpadding_tokens": 9245, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34183280000002014, "nonpadding_tokens": 13542, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34260343900018597, "nonpadding_tokens": 8658, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3448026690000461, "nonpadding_tokens": 12274, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3432519189998402, "nonpadding_tokens": 8379, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3445885589999307, "nonpadding_tokens": 1823, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3451519300001564, "nonpadding_tokens": 1520, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037844100006623194, "prefetch": {"data_wait_seconds": 0.02285428800173861, "consumed_batches": 2437, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2987, "tokens": 216951451, "loss_tokens": 72225, "loss": 2.208432632398754, "lr": 0.0002989593505690243, "optimizer_lrs": [0.01993062337126829, 0.0002989593505690243], "gradient_norm": 0.3932075500488281, "seconds": 0.2711217010000837, "nonpadding_tokens_per_second": 266393.2829190154, "ranks": [{"compute_seconds": 0.20475742899998295, "nonpadding_tokens": 2789, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2237674190000689, "nonpadding_tokens": 2921, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22261257000013757, "nonpadding_tokens": 9311, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22108158000014555, "nonpadding_tokens": 6138, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22015442200017787, "nonpadding_tokens": 20461, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22194284599981984, "nonpadding_tokens": 6424, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2193248810001478, "nonpadding_tokens": 13368, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22032314799980668, "nonpadding_tokens": 10813, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01925344899996162, "prefetch": {"data_wait_seconds": 0.022862033001686655, "consumed_batches": 2438, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2988, "tokens": 217012005, "loss_tokens": 60554, "loss": 2.1668291421706245, "lr": 0.0002989587113858562, "optimizer_lrs": [0.01993058075905708, 0.0002989587113858562], "gradient_norm": 0.46329089999198914, "seconds": 0.25464314300006663, "nonpadding_tokens_per_second": 237799.45254596608, "ranks": [{"compute_seconds": 0.2504037569999582, "nonpadding_tokens": 7044, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25195660899998984, "nonpadding_tokens": 5254, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24888306599996213, "nonpadding_tokens": 9688, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2508073419999164, "nonpadding_tokens": 8072, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24922895500003506, "nonpadding_tokens": 6038, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24901657500004148, "nonpadding_tokens": 8432, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2476705869999023, "nonpadding_tokens": 8562, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2523477820000153, "nonpadding_tokens": 7464, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036598299993784167, "prefetch": {"data_wait_seconds": 0.022871589001582038, "consumed_batches": 2439, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2989, "tokens": 217052843, "loss_tokens": 40838, "loss": 2.2035041658504335, "lr": 0.00029895828020632135, "optimizer_lrs": [0.01993055201375476, 0.00029895828020632135], "gradient_norm": 0.4478848874568939, "seconds": 0.2845715610001207, "nonpadding_tokens_per_second": 143506.96132978192, "ranks": [{"compute_seconds": 0.2812390209999194, "nonpadding_tokens": 10294, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2817146729998967, "nonpadding_tokens": 7869, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.278710467999872, "nonpadding_tokens": 3242, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28187110300018503, "nonpadding_tokens": 4169, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27928640499999347, "nonpadding_tokens": 5175, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28094397900008516, "nonpadding_tokens": 3383, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2822259639999629, "nonpadding_tokens": 2678, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27906531700000414, "nonpadding_tokens": 4028, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003801160000875825, "prefetch": {"data_wait_seconds": 0.02288145000166253, "consumed_batches": 2440, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2990, "tokens": 217132447, "loss_tokens": 79604, "loss": 2.688167562245616, "lr": 0.00029895743946847293, "optimizer_lrs": [0.019930495964564865, 0.00029895743946847293], "gradient_norm": 0.4707905948162079, "seconds": 0.2601200209999206, "nonpadding_tokens_per_second": 306027.962376738, "ranks": [{"compute_seconds": 0.25687330499999916, "nonpadding_tokens": 13957, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2571471409999049, "nonpadding_tokens": 4411, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25458833599986974, "nonpadding_tokens": 16885, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2567877999999837, "nonpadding_tokens": 9533, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2534885509999185, "nonpadding_tokens": 13771, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2548834629999419, "nonpadding_tokens": 9654, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25773372499998004, "nonpadding_tokens": 4459, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25352855799997087, "nonpadding_tokens": 6934, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038906100007807254, "prefetch": {"data_wait_seconds": 0.022890600001531425, "consumed_batches": 2441, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2991, "tokens": 217213877, "loss_tokens": 81430, "loss": 2.1126753806950753, "lr": 0.0002989565790957945, "optimizer_lrs": [0.0199304386063863, 0.0002989565790957945], "gradient_norm": 0.40545591711997986, "seconds": 0.3266085269999621, "nonpadding_tokens_per_second": 249319.88380085817, "ranks": [{"compute_seconds": 0.32417393900004754, "nonpadding_tokens": 14563, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32307117699997434, "nonpadding_tokens": 12942, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3232375380000576, "nonpadding_tokens": 18512, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3235313329998917, "nonpadding_tokens": 11271, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31910109399996145, "nonpadding_tokens": 17415, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3211232439998639, "nonpadding_tokens": 2934, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3235967430000528, "nonpadding_tokens": 1196, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32004666099987844, "nonpadding_tokens": 2597, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004535919999852922, "prefetch": {"data_wait_seconds": 0.02290243500142708, "consumed_batches": 2442, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2992, "tokens": 217273190, "loss_tokens": 59313, "loss": 1.9085446329219564, "lr": 0.00029895595218429056, "optimizer_lrs": [0.01993039681228604, 0.00029895595218429056], "gradient_norm": 0.4327543377876282, "seconds": 0.31299631100000624, "nonpadding_tokens_per_second": 189500.63599950486, "ranks": [{"compute_seconds": 0.3100640079999266, "nonpadding_tokens": 5421, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30722194099985245, "nonpadding_tokens": 3268, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3075233770000523, "nonpadding_tokens": 5726, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30806813399999555, "nonpadding_tokens": 3231, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31062147800003004, "nonpadding_tokens": 1864, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3094736840000678, "nonpadding_tokens": 11524, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30638945200007583, "nonpadding_tokens": 16975, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30712158699998326, "nonpadding_tokens": 11304, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037612200003422913, "prefetch": {"data_wait_seconds": 0.022911155001523298, "consumed_batches": 2443, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2993, "tokens": 217329982, "loss_tokens": 56792, "loss": 2.1397562871971405, "lr": 0.0002989553517428566, "optimizer_lrs": [0.019930356782857105, 0.0002989553517428566], "gradient_norm": 0.5102384090423584, "seconds": 0.27336563499989097, "nonpadding_tokens_per_second": 207751.0583948222, "ranks": [{"compute_seconds": 0.27103464800006805, "nonpadding_tokens": 4334, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26975208699991526, "nonpadding_tokens": 7900, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2688428489998387, "nonpadding_tokens": 3263, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26682607399993685, "nonpadding_tokens": 22648, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26522817099998974, "nonpadding_tokens": 5168, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2682049719999213, "nonpadding_tokens": 3255, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2649168009997993, "nonpadding_tokens": 4171, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2659248560000833, "nonpadding_tokens": 6053, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003573740000319958, "prefetch": {"data_wait_seconds": 0.022920626001408095, "consumed_batches": 2444, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2994, "tokens": 217385511, "loss_tokens": 55529, "loss": 2.3270606631219724, "lr": 0.00029895476448842204, "optimizer_lrs": [0.01993031763256147, 0.00029895476448842204], "gradient_norm": 0.4757835566997528, "seconds": 0.24671894699986296, "nonpadding_tokens_per_second": 225069.86461818372, "ranks": [{"compute_seconds": 0.24433259399984308, "nonpadding_tokens": 6629, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24118942499990226, "nonpadding_tokens": 10981, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2416218039998057, "nonpadding_tokens": 12674, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24419115399996372, "nonpadding_tokens": 2414, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24048450800000865, "nonpadding_tokens": 3803, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24324947000013708, "nonpadding_tokens": 5937, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24052165000011883, "nonpadding_tokens": 11124, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24118405399985932, "nonpadding_tokens": 1967, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040566400002717273, "prefetch": {"data_wait_seconds": 0.022929815001319298, "consumed_batches": 2445, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2995, "tokens": 217480147, "loss_tokens": 94636, "loss": 2.306132180142863, "lr": 0.0002989537632738445, "optimizer_lrs": [0.01993025088492297, 0.0002989537632738445], "gradient_norm": 0.45409712195396423, "seconds": 0.33393915400006335, "nonpadding_tokens_per_second": 283392.944092989, "ranks": [{"compute_seconds": 0.3306198380000751, "nonpadding_tokens": 17763, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.330538592000039, "nonpadding_tokens": 16489, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33043730700001106, "nonpadding_tokens": 4090, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32832241899996006, "nonpadding_tokens": 11770, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3268941219998851, "nonpadding_tokens": 10347, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32993007599998236, "nonpadding_tokens": 16654, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3267174000000068, "nonpadding_tokens": 7184, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3315528460000223, "nonpadding_tokens": 10339, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003607199998896249, "prefetch": {"data_wait_seconds": 0.022939131001251667, "consumed_batches": 2446, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2996, "tokens": 217557869, "loss_tokens": 77722, "loss": 2.210495017498263, "lr": 0.00029895294064619587, "optimizer_lrs": [0.019930196043079725, 0.00029895294064619587], "gradient_norm": 0.37854573130607605, "seconds": 0.2735956129999977, "nonpadding_tokens_per_second": 284076.1924059091, "ranks": [{"compute_seconds": 0.2704108759999144, "nonpadding_tokens": 15240, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27051688400001694, "nonpadding_tokens": 12609, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.270765836999999, "nonpadding_tokens": 8095, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2679676349998772, "nonpadding_tokens": 10028, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2659341589999258, "nonpadding_tokens": 16694, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26842589000011685, "nonpadding_tokens": 5492, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2658288619998075, "nonpadding_tokens": 5074, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.271112789999961, "nonpadding_tokens": 4490, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038726800016775087, "prefetch": {"data_wait_seconds": 0.022948896001210528, "consumed_batches": 2447, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2997, "tokens": 217618459, "loss_tokens": 60590, "loss": 2.2697722396435056, "lr": 0.00029895229912423336, "optimizer_lrs": [0.019930153274948893, 0.00029895229912423336], "gradient_norm": 0.40832799673080444, "seconds": 0.30979256999989957, "nonpadding_tokens_per_second": 195582.48282074564, "ranks": [{"compute_seconds": 0.30736304700008077, "nonpadding_tokens": 6694, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30651225399992654, "nonpadding_tokens": 7866, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30381294200014963, "nonpadding_tokens": 8532, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30599602999996023, "nonpadding_tokens": 14246, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30223508600010973, "nonpadding_tokens": 2929, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3049216410001918, "nonpadding_tokens": 7472, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3024738219999108, "nonpadding_tokens": 1638, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3072237460000906, "nonpadding_tokens": 11213, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003590619999158662, "prefetch": {"data_wait_seconds": 0.02295930500118004, "consumed_batches": 2448, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2998, "tokens": 217683039, "loss_tokens": 64580, "loss": 2.4599917253793744, "lr": 0.00029895161514102085, "optimizer_lrs": [0.019930107676068058, 0.00029895161514102085], "gradient_norm": 0.4768487811088562, "seconds": 0.3029450999999881, "nonpadding_tokens_per_second": 213173.9381161885, "ranks": [{"compute_seconds": 0.2970741280000766, "nonpadding_tokens": 12636, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2994597880001493, "nonpadding_tokens": 8018, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2994756169998709, "nonpadding_tokens": 15078, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2980073280000397, "nonpadding_tokens": 4556, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3004957360001299, "nonpadding_tokens": 2174, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2978081960000054, "nonpadding_tokens": 7126, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2960850340000434, "nonpadding_tokens": 6430, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.300629403999892, "nonpadding_tokens": 8562, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039139299997259513, "prefetch": {"data_wait_seconds": 0.022968672001027812, "consumed_batches": 2449, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 2999, "tokens": 217767203, "loss_tokens": 84164, "loss": 2.086882826980657, "lr": 0.0002989507234050054, "optimizer_lrs": [0.019930048227000362, 0.0002989507234050054], "gradient_norm": 0.39465221762657166, "seconds": 0.23826523999991878, "nonpadding_tokens_per_second": 353236.5862516441, "ranks": [{"compute_seconds": 0.19684010599985413, "nonpadding_tokens": 6545, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20877347499981624, "nonpadding_tokens": 5606, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20864101799998025, "nonpadding_tokens": 3958, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2068060050000895, "nonpadding_tokens": 5704, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.20547592100001566, "nonpadding_tokens": 16138, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20799189099989235, "nonpadding_tokens": 8238, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2045886559999417, "nonpadding_tokens": 16399, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20617569399996682, "nonpadding_tokens": 21576, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0031336889999238338, "prefetch": {"data_wait_seconds": 0.022977682001013022, "consumed_batches": 2450, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3000, "tokens": 217806656, "loss_tokens": 39453, "loss": 2.09238227289433, "lr": 0.0002989503052618371, "optimizer_lrs": [0.01993002035078914, 0.0002989503052618371], "gradient_norm": 0.4614887833595276, "seconds": 0.25091247900013514, "nonpadding_tokens_per_second": 157238.09416421552, "ranks": [{"compute_seconds": 0.24640953200014337, "nonpadding_tokens": 1577, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24870063200000914, "nonpadding_tokens": 1265, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24569191400019008, "nonpadding_tokens": 7171, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2456003399997826, "nonpadding_tokens": 3360, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24436244699995768, "nonpadding_tokens": 10118, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24721523099992737, "nonpadding_tokens": 4563, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2482690129997991, "nonpadding_tokens": 2281, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24517309899988504, "nonpadding_tokens": 9118, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003369879998444958, "prefetch": {"data_wait_seconds": 0.02298714800117807, "consumed_batches": 2451, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3001, "tokens": 217897542, "loss_tokens": 90886, "loss": 2.350858321138569, "lr": 0.00029894934168956706, "optimizer_lrs": [0.019929956112637804, 0.00029894934168956706], "gradient_norm": 0.35397276282310486, "seconds": 0.285536964999892, "nonpadding_tokens_per_second": 318298.5432377709, "ranks": [{"compute_seconds": 0.2806276860001162, "nonpadding_tokens": 9846, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28285146899997926, "nonpadding_tokens": 16514, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2822742430000744, "nonpadding_tokens": 13431, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2802124349998394, "nonpadding_tokens": 9743, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27859951200002797, "nonpadding_tokens": 5205, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2815212719999636, "nonpadding_tokens": 10740, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27875154700018356, "nonpadding_tokens": 16600, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2793029829999796, "nonpadding_tokens": 8807, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044173399987812445, "prefetch": {"data_wait_seconds": 0.02299743200133264, "consumed_batches": 2452, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3002, "tokens": 217946153, "loss_tokens": 48611, "loss": 2.221127773292053, "lr": 0.00029894882613559553, "optimizer_lrs": [0.019929921742373038, 0.00029894882613559553], "gradient_norm": 0.427645206451416, "seconds": 0.3159632889999102, "nonpadding_tokens_per_second": 153850.15187639033, "ranks": [{"compute_seconds": 0.2655793399999311, "nonpadding_tokens": 4739, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2735764829999425, "nonpadding_tokens": 5657, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27389364100008606, "nonpadding_tokens": 1743, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27368208899997626, "nonpadding_tokens": 3265, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27275310300001365, "nonpadding_tokens": 3111, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2738750500000151, "nonpadding_tokens": 14793, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27277190099994186, "nonpadding_tokens": 11853, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27339806700001645, "nonpadding_tokens": 3450, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.018414514999903986, "prefetch": {"data_wait_seconds": 0.023008338001275774, "consumed_batches": 2453, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3003, "tokens": 218004120, "loss_tokens": 57967, "loss": 2.569749275449825, "lr": 0.0002989482111899754, "optimizer_lrs": [0.01992988074599836, 0.0002989482111899754], "gradient_norm": 3.77805495262146, "seconds": 0.3317494220000299, "nonpadding_tokens_per_second": 174731.27654761905, "ranks": [{"compute_seconds": 0.3287979209999321, "nonpadding_tokens": 8756, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32891301800009387, "nonpadding_tokens": 7382, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32882234800013066, "nonpadding_tokens": 6060, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3264961960001074, "nonpadding_tokens": 6009, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32931113499989806, "nonpadding_tokens": 12474, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3264696640001148, "nonpadding_tokens": 11582, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.329059523000069, "nonpadding_tokens": 3189, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32539009300012367, "nonpadding_tokens": 2515, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037638899993908126, "prefetch": {"data_wait_seconds": 0.023017573001155256, "consumed_batches": 2454, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3004, "tokens": 218059675, "loss_tokens": 55555, "loss": 2.3780192489424894, "lr": 0.0002989476216640761, "optimizer_lrs": [0.01992984144427174, 0.0002989476216640761], "gradient_norm": 0.7382053136825562, "seconds": 0.3382973299999321, "nonpadding_tokens_per_second": 164219.44565749646, "ranks": [{"compute_seconds": 0.335923553999919, "nonpadding_tokens": 9368, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3324616969998715, "nonpadding_tokens": 4327, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33278955500009033, "nonpadding_tokens": 3756, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3331855250000899, "nonpadding_tokens": 4932, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33144899199987776, "nonpadding_tokens": 9561, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3333017649999874, "nonpadding_tokens": 2936, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33561624399999346, "nonpadding_tokens": 11428, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3321455860000242, "nonpadding_tokens": 9247, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036409800009096216, "prefetch": {"data_wait_seconds": 0.023027210001103526, "consumed_batches": 2455, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3005, "tokens": 218105824, "loss_tokens": 46149, "loss": 2.365598557119331, "lr": 0.00029894713182552894, "optimizer_lrs": [0.019929808788368596, 0.00029894713182552894], "gradient_norm": 0.5290102958679199, "seconds": 0.2621411119998811, "nonpadding_tokens_per_second": 176046.4035874729, "ranks": [{"compute_seconds": 0.2579808990001311, "nonpadding_tokens": 2088, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2590118309999525, "nonpadding_tokens": 10641, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2561096729998553, "nonpadding_tokens": 8233, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25972969200006446, "nonpadding_tokens": 2346, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25969215899999654, "nonpadding_tokens": 9685, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25792196099996545, "nonpadding_tokens": 3601, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2549098790000244, "nonpadding_tokens": 1537, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2553838950000227, "nonpadding_tokens": 8018, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038087100006123364, "prefetch": {"data_wait_seconds": 0.02303678500102535, "consumed_batches": 2456, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3006, "tokens": 218177459, "loss_tokens": 71635, "loss": 2.528256526139457, "lr": 0.00029894637124648214, "optimizer_lrs": [0.01992975808309881, 0.00029894637124648214], "gradient_norm": 0.44548627734184265, "seconds": 0.3004377800000384, "nonpadding_tokens_per_second": 238435.3925128552, "ranks": [{"compute_seconds": 0.29715234199989027, "nonpadding_tokens": 5040, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29331200000001445, "nonpadding_tokens": 11897, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29585100299982514, "nonpadding_tokens": 14313, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29728859399983776, "nonpadding_tokens": 5854, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2965752130000965, "nonpadding_tokens": 5295, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2981264239999746, "nonpadding_tokens": 12530, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2967554399999699, "nonpadding_tokens": 13964, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2962147319999531, "nonpadding_tokens": 2742, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004312629998821649, "prefetch": {"data_wait_seconds": 0.02304679100093665, "consumed_batches": 2457, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3007, "tokens": 218257273, "loss_tokens": 79814, "loss": 2.3055107030094972, "lr": 0.00029894552350536667, "optimizer_lrs": [0.019929701567024446, 0.00029894552350536667], "gradient_norm": 0.46969074010849, "seconds": 0.42290309400004844, "nonpadding_tokens_per_second": 188728.81549547342, "ranks": [{"compute_seconds": 0.4204506170001423, "nonpadding_tokens": 17168, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.42037038800003756, "nonpadding_tokens": 3532, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4165850300000784, "nonpadding_tokens": 2663, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4179324120000274, "nonpadding_tokens": 4847, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.4159448939999493, "nonpadding_tokens": 5810, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.41884452299996155, "nonpadding_tokens": 21207, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.416886599999998, "nonpadding_tokens": 11314, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4203927940000085, "nonpadding_tokens": 13273, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037568199991255824, "prefetch": {"data_wait_seconds": 0.023055225000874202, "consumed_batches": 2458, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3008, "tokens": 218352510, "loss_tokens": 95237, "loss": 2.379191024759285, "lr": 0.00029894451150515766, "optimizer_lrs": [0.019929634100343846, 0.00029894451150515766], "gradient_norm": 0.3778253197669983, "seconds": 0.3041108580000582, "nonpadding_tokens_per_second": 313165.4049655201, "ranks": [{"compute_seconds": 0.28029696600015086, "nonpadding_tokens": 15239, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2885569489999398, "nonpadding_tokens": 5508, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28415217699989626, "nonpadding_tokens": 2030, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2845597739999448, "nonpadding_tokens": 4560, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28728143700004694, "nonpadding_tokens": 17771, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2847152819999792, "nonpadding_tokens": 20122, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2870105029999195, "nonpadding_tokens": 18214, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2835671909999746, "nonpadding_tokens": 11793, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004784509999353759, "prefetch": {"data_wait_seconds": 0.023063720000848775, "consumed_batches": 2459, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3009, "tokens": 218415489, "loss_tokens": 62979, "loss": 2.4158856523603105, "lr": 0.00029894384201689726, "optimizer_lrs": [0.019929589467793152, 0.00029894384201689726], "gradient_norm": 0.4928950369358063, "seconds": 0.29700919400011117, "nonpadding_tokens_per_second": 212043.94096964024, "ranks": [{"compute_seconds": 0.293976444000009, "nonpadding_tokens": 11794, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29443080100008956, "nonpadding_tokens": 9417, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2913473040000554, "nonpadding_tokens": 3020, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2943209340000976, "nonpadding_tokens": 11123, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.290636091000124, "nonpadding_tokens": 2028, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29203804700000546, "nonpadding_tokens": 10363, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2946060669999042, "nonpadding_tokens": 9536, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2910093489999781, "nonpadding_tokens": 5698, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003725290000602399, "prefetch": {"data_wait_seconds": 0.023072560001040188, "consumed_batches": 2460, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3010, "tokens": 218475918, "loss_tokens": 60429, "loss": 2.398874867613232, "lr": 0.0002989431994372499, "optimizer_lrs": [0.019929546629149995, 0.0002989431994372499], "gradient_norm": 0.5096017122268677, "seconds": 0.34584317799999553, "nonpadding_tokens_per_second": 174729.4838934217, "ranks": [{"compute_seconds": 0.3427467350002189, "nonpadding_tokens": 13918, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3433439460000045, "nonpadding_tokens": 3404, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3397656120000647, "nonpadding_tokens": 2847, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3404206739999154, "nonpadding_tokens": 8378, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3432287720002023, "nonpadding_tokens": 9301, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.341497340999922, "nonpadding_tokens": 4241, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3428629570000794, "nonpadding_tokens": 10710, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3435485530001188, "nonpadding_tokens": 7630, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033648400017227686, "prefetch": {"data_wait_seconds": 0.02308047000106228, "consumed_batches": 2461, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3011, "tokens": 218578785, "loss_tokens": 102867, "loss": 2.2032313266159216, "lr": 0.000298942105139903, "optimizer_lrs": [0.019929473675993535, 0.000298942105139903], "gradient_norm": 0.4130996763706207, "seconds": 0.2932489310001074, "nonpadding_tokens_per_second": 350783.8874268985, "ranks": [{"compute_seconds": 0.2887062249999417, "nonpadding_tokens": 15769, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2908447370000431, "nonpadding_tokens": 11393, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2879813740000827, "nonpadding_tokens": 15218, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29070820400011144, "nonpadding_tokens": 18352, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29096194700014166, "nonpadding_tokens": 14269, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28824389999999767, "nonpadding_tokens": 9483, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.290828503000057, "nonpadding_tokens": 7065, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2877774020000743, "nonpadding_tokens": 11318, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003753650000817288, "prefetch": {"data_wait_seconds": 0.02308970900089946, "consumed_batches": 2462, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3012, "tokens": 218635601, "loss_tokens": 56816, "loss": 2.213864987415517, "lr": 0.0002989415004904876, "optimizer_lrs": [0.019929433366032506, 0.0002989415004904876], "gradient_norm": 0.6688148379325867, "seconds": 0.23485732199992526, "nonpadding_tokens_per_second": 241917.0904112501, "ranks": [{"compute_seconds": 0.23025158500013276, "nonpadding_tokens": 3053, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2326541840000118, "nonpadding_tokens": 5277, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22966788200005794, "nonpadding_tokens": 3036, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23257897799999228, "nonpadding_tokens": 8468, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22816337899985228, "nonpadding_tokens": 4543, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23015529099984633, "nonpadding_tokens": 7412, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.228913982999984, "nonpadding_tokens": 11971, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22899204200007262, "nonpadding_tokens": 13056, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003366559999449237, "prefetch": {"data_wait_seconds": 0.02309797000107494, "consumed_batches": 2463, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3013, "tokens": 218696230, "loss_tokens": 60629, "loss": 2.3314255554272707, "lr": 0.00029894085507244144, "optimizer_lrs": [0.019929390338162763, 0.00029894085507244144], "gradient_norm": 0.47831225395202637, "seconds": 0.26754943900004946, "nonpadding_tokens_per_second": 226608.58578734973, "ranks": [{"compute_seconds": 0.26368640900000173, "nonpadding_tokens": 5892, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.260782995999989, "nonpadding_tokens": 9336, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2610749540001507, "nonpadding_tokens": 7029, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2637947969999459, "nonpadding_tokens": 3394, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26513145300009455, "nonpadding_tokens": 10892, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26127389199996287, "nonpadding_tokens": 6003, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2600796540000374, "nonpadding_tokens": 2307, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2605183499999839, "nonpadding_tokens": 15776, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003863669999191188, "prefetch": {"data_wait_seconds": 0.02310796900106027, "consumed_batches": 2464, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3014, "tokens": 218774261, "loss_tokens": 78031, "loss": 2.252238693596135, "lr": 0.00029894002411532793, "optimizer_lrs": [0.019929334941021865, 0.00029894002411532793], "gradient_norm": 0.5107743144035339, "seconds": 0.31432713399999557, "nonpadding_tokens_per_second": 248247.73797607017, "ranks": [{"compute_seconds": 0.295924623000019, "nonpadding_tokens": 21941, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3008323429999109, "nonpadding_tokens": 7835, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2981409399999393, "nonpadding_tokens": 11259, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30086418000018966, "nonpadding_tokens": 4424, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29730902999995124, "nonpadding_tokens": 9781, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29819054599988704, "nonpadding_tokens": 2753, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30064606499990987, "nonpadding_tokens": 6659, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29731901300010577, "nonpadding_tokens": 13379, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004369299999780196, "prefetch": {"data_wait_seconds": 0.02311625100124104, "consumed_batches": 2465, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3015, "tokens": 218831596, "loss_tokens": 57335, "loss": 1.918565503183047, "lr": 0.00029893941334439524, "optimizer_lrs": [0.019929294222959686, 0.00029893941334439524], "gradient_norm": 0.5196103453636169, "seconds": 0.2523470250000628, "nonpadding_tokens_per_second": 227206.9583542177, "ranks": [{"compute_seconds": 0.24824002400009704, "nonpadding_tokens": 8053, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25014376700005414, "nonpadding_tokens": 9861, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24705935399992995, "nonpadding_tokens": 4270, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24910428300017884, "nonpadding_tokens": 8700, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24567271599994456, "nonpadding_tokens": 2281, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24818468800003757, "nonpadding_tokens": 4169, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24574821599981078, "nonpadding_tokens": 10995, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24650296799995886, "nonpadding_tokens": 9006, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003377330001512746, "prefetch": {"data_wait_seconds": 0.02312437600126316, "consumed_batches": 2466, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3016, "tokens": 218873216, "loss_tokens": 41620, "loss": 2.423854779853436, "lr": 0.0002989389698704346, "optimizer_lrs": [0.019929264658028977, 0.0002989389698704346], "gradient_norm": 0.44807660579681396, "seconds": 0.2313474000000042, "nonpadding_tokens_per_second": 179902.60534589645, "ranks": [{"compute_seconds": 0.2291554090002137, "nonpadding_tokens": 4597, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22910213799991652, "nonpadding_tokens": 6322, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22851224400005776, "nonpadding_tokens": 2105, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22840766799981793, "nonpadding_tokens": 4248, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22534330500002397, "nonpadding_tokens": 5811, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22801248199994006, "nonpadding_tokens": 7617, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22500738500002626, "nonpadding_tokens": 4883, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22621695500015448, "nonpadding_tokens": 6037, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003811309998127399, "prefetch": {"data_wait_seconds": 0.02313309900132765, "consumed_batches": 2467, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3017, "tokens": 218921087, "loss_tokens": 47871, "loss": 2.3412217208748514, "lr": 0.0002989384596759628, "optimizer_lrs": [0.01992923064506419, 0.0002989384596759628], "gradient_norm": 0.41314148902893066, "seconds": 0.29284108100000594, "nonpadding_tokens_per_second": 163470.9168417495, "ranks": [{"compute_seconds": 0.2895951010000317, "nonpadding_tokens": 4482, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28677881699991303, "nonpadding_tokens": 4597, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28909000899989223, "nonpadding_tokens": 3618, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2896029380001437, "nonpadding_tokens": 8889, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29045721700003924, "nonpadding_tokens": 2105, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29059059799988063, "nonpadding_tokens": 11391, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2876219590000346, "nonpadding_tokens": 2986, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2886995129999832, "nonpadding_tokens": 9803, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038513199979206547, "prefetch": {"data_wait_seconds": 0.023140938001418476, "consumed_batches": 2468, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3018, "tokens": 218978582, "loss_tokens": 57495, "loss": 2.4019469084268197, "lr": 0.0002989378467503902, "optimizer_lrs": [0.01992918978335935, 0.0002989378467503902], "gradient_norm": 0.9124189615249634, "seconds": 0.36382239199997457, "nonpadding_tokens_per_second": 158030.40512141984, "ranks": [{"compute_seconds": 0.3612637050000558, "nonpadding_tokens": 12916, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3594589869999254, "nonpadding_tokens": 9712, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3600575519999438, "nonpadding_tokens": 5333, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3602965800000675, "nonpadding_tokens": 3374, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.35912983699995493, "nonpadding_tokens": 2555, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3616220339999927, "nonpadding_tokens": 12988, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3590649459999895, "nonpadding_tokens": 2162, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3597301239999524, "nonpadding_tokens": 8455, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036134699985268526, "prefetch": {"data_wait_seconds": 0.023149914001535876, "consumed_batches": 2469, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3019, "tokens": 219019833, "loss_tokens": 41251, "loss": 2.3184691734745826, "lr": 0.0002989374068854552, "optimizer_lrs": [0.01992916045903035, 0.0002989374068854552], "gradient_norm": 0.4335218071937561, "seconds": 0.30579204699984075, "nonpadding_tokens_per_second": 134898.86478316906, "ranks": [{"compute_seconds": 0.2990131909998581, "nonpadding_tokens": 12201, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3011187060001248, "nonpadding_tokens": 2621, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2972878569999011, "nonpadding_tokens": 6562, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.298525305000112, "nonpadding_tokens": 4100, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30342167699996025, "nonpadding_tokens": 4812, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3008125229998768, "nonpadding_tokens": 5337, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29659931400010464, "nonpadding_tokens": 2970, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29741717999991124, "nonpadding_tokens": 2648, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034318100006203167, "prefetch": {"data_wait_seconds": 0.0231589460015158, "consumed_batches": 2470, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3020, "tokens": 219072684, "loss_tokens": 52851, "loss": 2.3611779755822973, "lr": 0.00029893684319562197, "optimizer_lrs": [0.019929122879708133, 0.00029893684319562197], "gradient_norm": 0.4366741180419922, "seconds": 0.31088569300004565, "nonpadding_tokens_per_second": 170001.39018939107, "ranks": [{"compute_seconds": 0.28991869700007555, "nonpadding_tokens": 5185, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2926349439999285, "nonpadding_tokens": 3804, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28939752400015095, "nonpadding_tokens": 7950, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2920014709998213, "nonpadding_tokens": 14432, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28805979200001275, "nonpadding_tokens": 6716, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28975198800003454, "nonpadding_tokens": 2399, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29210988299996643, "nonpadding_tokens": 3083, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28863779200014505, "nonpadding_tokens": 9282, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.005477283000118405, "prefetch": {"data_wait_seconds": 0.023167485001479804, "consumed_batches": 2471, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3021, "tokens": 219156595, "loss_tokens": 83911, "loss": 2.457233705950352, "lr": 0.0002989359479251148, "optimizer_lrs": [0.019929063195007656, 0.0002989359479251148], "gradient_norm": 0.4282556474208832, "seconds": 0.26977455200017175, "nonpadding_tokens_per_second": 311041.19857808744, "ranks": [{"compute_seconds": 0.2669809609999447, "nonpadding_tokens": 15675, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2651599329999499, "nonpadding_tokens": 4202, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2659351649999735, "nonpadding_tokens": 3529, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26293882000004487, "nonpadding_tokens": 25031, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26288172900012796, "nonpadding_tokens": 14568, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2643723820001469, "nonpadding_tokens": 3826, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2667780980000316, "nonpadding_tokens": 10110, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26742857500016726, "nonpadding_tokens": 6970, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034481799980312644, "prefetch": {"data_wait_seconds": 0.023177452001348797, "consumed_batches": 2472, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3022, "tokens": 219263791, "loss_tokens": 107196, "loss": 2.2866011791484757, "lr": 0.0002989348036743064, "optimizer_lrs": [0.019928986911620428, 0.0002989348036743064], "gradient_norm": 0.37133097648620605, "seconds": 0.24686669099992287, "nonpadding_tokens_per_second": 434226.2601965751, "ranks": [{"compute_seconds": 0.24399044299980233, "nonpadding_tokens": 2836, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2417661560000397, "nonpadding_tokens": 21692, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23890752299985252, "nonpadding_tokens": 12232, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23866167300002417, "nonpadding_tokens": 11212, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24411215600002834, "nonpadding_tokens": 18453, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2406521350001185, "nonpadding_tokens": 14338, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23782657199990354, "nonpadding_tokens": 15499, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24440024500017898, "nonpadding_tokens": 10934, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003792549998706818, "prefetch": {"data_wait_seconds": 0.02318650900129171, "consumed_batches": 2473, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3023, "tokens": 219322785, "loss_tokens": 58994, "loss": 2.3840639514187885, "lr": 0.0002989341736886623, "optimizer_lrs": [0.019928944912577486, 0.0002989341736886623], "gradient_norm": 0.7478545308113098, "seconds": 0.27750379100007194, "nonpadding_tokens_per_second": 212588.0867695415, "ranks": [{"compute_seconds": 0.27493330699985563, "nonpadding_tokens": 16914, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2750531479998699, "nonpadding_tokens": 2276, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.274629564000179, "nonpadding_tokens": 5837, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27502933000005214, "nonpadding_tokens": 6562, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27150769299987587, "nonpadding_tokens": 2086, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2730054310000014, "nonpadding_tokens": 10989, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27103121200002533, "nonpadding_tokens": 7361, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2720444750000297, "nonpadding_tokens": 6969, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004190570000446314, "prefetch": {"data_wait_seconds": 0.023195515001319222, "consumed_batches": 2474, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3024, "tokens": 219415343, "loss_tokens": 92558, "loss": 2.24941286814754, "lr": 0.00029893318490564774, "optimizer_lrs": [0.01992887899370985, 0.00029893318490564774], "gradient_norm": 0.4182944893836975, "seconds": 0.28317320199994356, "nonpadding_tokens_per_second": 326860.02540600026, "ranks": [{"compute_seconds": 0.27844675699998334, "nonpadding_tokens": 5162, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27793841500010785, "nonpadding_tokens": 12844, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2806015930000285, "nonpadding_tokens": 4149, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27641771699995843, "nonpadding_tokens": 13823, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27462439200007793, "nonpadding_tokens": 5200, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2783012240001881, "nonpadding_tokens": 4396, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27487926699996024, "nonpadding_tokens": 28166, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.275572221000175, "nonpadding_tokens": 18818, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038319100008266105, "prefetch": {"data_wait_seconds": 0.02320533000120406, "consumed_batches": 2475, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3025, "tokens": 219462153, "loss_tokens": 46810, "loss": 2.126479718543046, "lr": 0.00029893268466768763, "optimizer_lrs": [0.01992884564451251, 0.00029893268466768763], "gradient_norm": 0.3940410315990448, "seconds": 0.271485728000016, "nonpadding_tokens_per_second": 172421.5867435847, "ranks": [{"compute_seconds": 0.2646347050001623, "nonpadding_tokens": 5040, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2670090829999481, "nonpadding_tokens": 5793, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26406977000010556, "nonpadding_tokens": 3258, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26902637399984997, "nonpadding_tokens": 5475, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26326255000003584, "nonpadding_tokens": 4063, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26656419999994796, "nonpadding_tokens": 10523, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2630527780002012, "nonpadding_tokens": 4865, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26343939400021554, "nonpadding_tokens": 7793, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004455220000636473, "prefetch": {"data_wait_seconds": 0.023216355001252964, "consumed_batches": 2476, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3026, "tokens": 219513426, "loss_tokens": 51273, "loss": 2.376688723841008, "lr": 0.00029893213660157875, "optimizer_lrs": [0.01992880910677192, 0.00029893213660157875], "gradient_norm": 0.4356285035610199, "seconds": 0.3124251460001233, "nonpadding_tokens_per_second": 164112.9104248855, "ranks": [{"compute_seconds": 0.30473618299993177, "nonpadding_tokens": 11520, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3073962629998732, "nonpadding_tokens": 2631, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3069105279998894, "nonpadding_tokens": 5420, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30422423599998183, "nonpadding_tokens": 6314, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3031258720000096, "nonpadding_tokens": 7983, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3059408110000277, "nonpadding_tokens": 6109, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3031905089999327, "nonpadding_tokens": 5008, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3040155579999464, "nonpadding_tokens": 6288, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003880340000250726, "prefetch": {"data_wait_seconds": 0.02322518900105308, "consumed_batches": 2477, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3027, "tokens": 219575598, "loss_tokens": 62172, "loss": 2.5396850169690537, "lr": 0.0002989314718461787, "optimizer_lrs": [0.01992876478974525, 0.0002989314718461787], "gradient_norm": 0.46917474269866943, "seconds": 0.8493366959999094, "nonpadding_tokens_per_second": 73200.65210040876, "ranks": [{"compute_seconds": 0.8455794080000487, "nonpadding_tokens": 12408, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.8438008950001858, "nonpadding_tokens": 8494, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.8461057210001854, "nonpadding_tokens": 3489, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.8414843300001849, "nonpadding_tokens": 5925, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.8397264870000072, "nonpadding_tokens": 7110, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.8445921350000845, "nonpadding_tokens": 3414, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.846864554999911, "nonpadding_tokens": 9956, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.8407791820000057, "nonpadding_tokens": 11376, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003833949999716424, "prefetch": {"data_wait_seconds": 0.02323483000095905, "consumed_batches": 2478, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3028, "tokens": 219647035, "loss_tokens": 71437, "loss": 2.0838713219340117, "lr": 0.00029893070777319225, "optimizer_lrs": [0.01992871385154615, 0.00029893070777319225], "gradient_norm": 0.39460548758506775, "seconds": 0.2816983909999635, "nonpadding_tokens_per_second": 253593.92272854428, "ranks": [{"compute_seconds": 0.27515133199995034, "nonpadding_tokens": 4096, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2753240790000291, "nonpadding_tokens": 10680, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27323387799992815, "nonpadding_tokens": 6521, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27331653999999617, "nonpadding_tokens": 8412, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2791132379998089, "nonpadding_tokens": 2174, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2762129449999975, "nonpadding_tokens": 13948, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2719946350000555, "nonpadding_tokens": 12818, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2792636680001124, "nonpadding_tokens": 12788, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003818810000666417, "prefetch": {"data_wait_seconds": 0.023243021000965314, "consumed_batches": 2479, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3029, "tokens": 219690325, "loss_tokens": 43290, "loss": 2.0460807562370062, "lr": 0.00029893024462139794, "optimizer_lrs": [0.019928682974759863, 0.00029893024462139794], "gradient_norm": 0.4091803729534149, "seconds": 0.3064658439998311, "nonpadding_tokens_per_second": 141255.54559360244, "ranks": [{"compute_seconds": 0.3041342369999711, "nonpadding_tokens": 4395, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30376812500003325, "nonpadding_tokens": 5595, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30063048399983927, "nonpadding_tokens": 4156, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30136233600001106, "nonpadding_tokens": 3331, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29954580200001146, "nonpadding_tokens": 9431, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3023924369999804, "nonpadding_tokens": 4568, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2991081029999805, "nonpadding_tokens": 2591, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30041049200008274, "nonpadding_tokens": 9223, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003427209999244951, "prefetch": {"data_wait_seconds": 0.023251977000839474, "consumed_batches": 2480, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3030, "tokens": 219757052, "loss_tokens": 66727, "loss": 2.314686849026631, "lr": 0.00029892953052578934, "optimizer_lrs": [0.019928635368385957, 0.00029892953052578934], "gradient_norm": 0.392921507358551, "seconds": 0.24679760100002568, "nonpadding_tokens_per_second": 270371.34773442574, "ranks": [{"compute_seconds": 0.24251815499997065, "nonpadding_tokens": 2057, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24041774400006943, "nonpadding_tokens": 23214, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23965860299995256, "nonpadding_tokens": 2259, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2437582879999809, "nonpadding_tokens": 6191, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24436813700003768, "nonpadding_tokens": 15493, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24128230800010897, "nonpadding_tokens": 10428, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23954394199995477, "nonpadding_tokens": 3531, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2393008159999681, "nonpadding_tokens": 3554, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038535099997716316, "prefetch": {"data_wait_seconds": 0.023260364000819322, "consumed_batches": 2481, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3031, "tokens": 219864956, "loss_tokens": 107904, "loss": 3.586240431309312, "lr": 0.0002989283752620959, "optimizer_lrs": [0.019928558350806393, 0.0002989283752620959], "gradient_norm": 2.3611366748809814, "seconds": 0.34272144000010485, "nonpadding_tokens_per_second": 314844.6154987181, "ranks": [{"compute_seconds": 0.337999487000161, "nonpadding_tokens": 4608, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33589690700000574, "nonpadding_tokens": 4021, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3388423739997961, "nonpadding_tokens": 5721, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33753778799996326, "nonpadding_tokens": 14389, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33561509800006206, "nonpadding_tokens": 18557, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33836255999995046, "nonpadding_tokens": 12354, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3394831539999359, "nonpadding_tokens": 31035, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34034292399996957, "nonpadding_tokens": 17219, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034923699990940804, "prefetch": {"data_wait_seconds": 0.02326941300084684, "consumed_batches": 2482, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3032, "tokens": 219986240, "loss_tokens": 121284, "loss": 3.472066853418423, "lr": 0.0002989270760059699, "optimizer_lrs": [0.01992847173373133, 0.0002989270760059699], "gradient_norm": 1.8543323278427124, "seconds": 0.27456065299998045, "nonpadding_tokens_per_second": 441738.4598805155, "ranks": [{"compute_seconds": 0.25157481600012943, "nonpadding_tokens": 1364, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2573237540000264, "nonpadding_tokens": 21212, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25806956400015224, "nonpadding_tokens": 16299, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25812300399979904, "nonpadding_tokens": 10071, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25686449899990293, "nonpadding_tokens": 19865, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2581840360001024, "nonpadding_tokens": 18856, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2570417960000668, "nonpadding_tokens": 12227, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25729490200001237, "nonpadding_tokens": 21390, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003449910000199452, "prefetch": {"data_wait_seconds": 0.023277401000726172, "consumed_batches": 2483, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3033, "tokens": 220169641, "loss_tokens": 183401, "loss": 3.906564031820982, "lr": 0.0002989251098317018, "optimizer_lrs": [0.019928340655446786, 0.0002989251098317018], "gradient_norm": 3.692368984222412, "seconds": 0.322743102000004, "nonpadding_tokens_per_second": 568256.9166110256, "ranks": [{"compute_seconds": 0.31692557599990323, "nonpadding_tokens": 17030, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31616105400007655, "nonpadding_tokens": 21434, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3148435289999725, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31524056299986114, "nonpadding_tokens": 17560, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32041427299986935, "nonpadding_tokens": 26202, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31676826400007485, "nonpadding_tokens": 22015, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31908145000011245, "nonpadding_tokens": 21165, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3142047520000233, "nonpadding_tokens": 25227, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004002819998731866, "prefetch": {"data_wait_seconds": 0.02328586700078006, "consumed_batches": 2484, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3034, "tokens": 220320866, "loss_tokens": 151225, "loss": 3.4033061249793355, "lr": 0.0002989234872559076, "optimizer_lrs": [0.019928232483727173, 0.0002989234872559076], "gradient_norm": 2.167292356491089, "seconds": 0.39405526800010193, "nonpadding_tokens_per_second": 383765.9645244506, "ranks": [{"compute_seconds": 0.3884251790000235, "nonpadding_tokens": 18838, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.39049946300019656, "nonpadding_tokens": 16666, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3910445050000817, "nonpadding_tokens": 26429, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.38762522199999694, "nonpadding_tokens": 20694, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.386271134000026, "nonpadding_tokens": 10609, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38936931700004607, "nonpadding_tokens": 13084, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3916496950000692, "nonpadding_tokens": 19597, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3868050139999468, "nonpadding_tokens": 25308, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038551400007236225, "prefetch": {"data_wait_seconds": 0.023293926000818033, "consumed_batches": 2485, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3035, "tokens": 220389719, "loss_tokens": 68853, "loss": 2.507206022250301, "lr": 0.00029892274809058375, "optimizer_lrs": [0.019928183206038917, 0.00029892274809058375], "gradient_norm": 0.59967440366745, "seconds": 0.3289779350000117, "nonpadding_tokens_per_second": 209293.67192969203, "ranks": [{"compute_seconds": 0.32284483800003727, "nonpadding_tokens": 9792, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32338000400000055, "nonpadding_tokens": 7175, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32507599399991705, "nonpadding_tokens": 13993, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32589811900015775, "nonpadding_tokens": 2412, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31899554899996474, "nonpadding_tokens": 3296, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.322027541000125, "nonpadding_tokens": 2960, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32600027100011175, "nonpadding_tokens": 10553, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32633981699996184, "nonpadding_tokens": 18672, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033749299996088666, "prefetch": {"data_wait_seconds": 0.023302947000914287, "consumed_batches": 2486, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3036, "tokens": 220475910, "loss_tokens": 86191, "loss": 2.203372814446984, "lr": 0.00029892182243857813, "optimizer_lrs": [0.01992812149590521, 0.00029892182243857813], "gradient_norm": 0.4730701446533203, "seconds": 0.4674326699998801, "nonpadding_tokens_per_second": 184392.33184112294, "ranks": [{"compute_seconds": 0.4624054139999316, "nonpadding_tokens": 22259, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.4611710239998956, "nonpadding_tokens": 7676, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.45996649400012757, "nonpadding_tokens": 1707, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4647016110000095, "nonpadding_tokens": 7145, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.4649542919999021, "nonpadding_tokens": 11153, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.46173859100008485, "nonpadding_tokens": 18127, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.45858956200004286, "nonpadding_tokens": 13503, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.45978786199998467, "nonpadding_tokens": 4621, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004167930001131026, "prefetch": {"data_wait_seconds": 0.02331350900085738, "consumed_batches": 2487, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3037, "tokens": 220519490, "loss_tokens": 43580, "loss": 2.6850241294745296, "lr": 0.0002989213542585613, "optimizer_lrs": [0.019928090283904087, 0.0002989213542585613], "gradient_norm": 0.4716399610042572, "seconds": 0.21762431900015144, "nonpadding_tokens_per_second": 200253.35495694153, "ranks": [{"compute_seconds": 0.19107437200000277, "nonpadding_tokens": 3959, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19749455999999554, "nonpadding_tokens": 9234, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.19368535099988549, "nonpadding_tokens": 2150, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.19989181100004316, "nonpadding_tokens": 4372, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.19335010800000418, "nonpadding_tokens": 2822, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19564691599998696, "nonpadding_tokens": 2455, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.1928034000000025, "nonpadding_tokens": 16819, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20052126400014458, "nonpadding_tokens": 1769, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000560771999971621, "prefetch": {"data_wait_seconds": 0.023322323000869005, "consumed_batches": 2488, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3038, "tokens": 220599959, "loss_tokens": 80469, "loss": 1.930303711677789, "lr": 0.0002989204895139521, "optimizer_lrs": [0.019928032634263473, 0.0002989204895139521], "gradient_norm": 0.5909962058067322, "seconds": 0.27373127500004557, "nonpadding_tokens_per_second": 293970.7930706369, "ranks": [{"compute_seconds": 0.2708853780000027, "nonpadding_tokens": 6168, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2672667060001004, "nonpadding_tokens": 19421, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2700616929998887, "nonpadding_tokens": 11933, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27044376300000295, "nonpadding_tokens": 6780, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27115044199990734, "nonpadding_tokens": 4403, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26813973699995586, "nonpadding_tokens": 5711, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2708051010001782, "nonpadding_tokens": 11642, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2641170910001165, "nonpadding_tokens": 14411, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047087600000850216, "prefetch": {"data_wait_seconds": 0.023334106000859833, "consumed_batches": 2489, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3039, "tokens": 220674937, "loss_tokens": 74978, "loss": 2.470784555136173, "lr": 0.0002989196834667362, "optimizer_lrs": [0.019927978897782413, 0.0002989196834667362], "gradient_norm": 0.5792665481567383, "seconds": 0.3312265459999253, "nonpadding_tokens_per_second": 226364.70689162973, "ranks": [{"compute_seconds": 0.3277175769999303, "nonpadding_tokens": 10234, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32681488999992325, "nonpadding_tokens": 19620, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3238291669999853, "nonpadding_tokens": 10501, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32399298500013174, "nonpadding_tokens": 6799, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3286833980000665, "nonpadding_tokens": 11208, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32547809099992264, "nonpadding_tokens": 5328, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32308589500007656, "nonpadding_tokens": 3640, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3281457089999549, "nonpadding_tokens": 7648, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040990799993778637, "prefetch": {"data_wait_seconds": 0.023344271000908634, "consumed_batches": 2490, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3040, "tokens": 220768566, "loss_tokens": 93629, "loss": 2.3513493081737495, "lr": 0.00029891867649221355, "optimizer_lrs": [0.019927911766147572, 0.00029891867649221355], "gradient_norm": 0.42728087306022644, "seconds": 0.2795311219999803, "nonpadding_tokens_per_second": 334950.18132545036, "ranks": [{"compute_seconds": 0.273003676999906, "nonpadding_tokens": 10363, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.272939641999983, "nonpadding_tokens": 11807, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2721403259999988, "nonpadding_tokens": 2056, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.276128430999961, "nonpadding_tokens": 17175, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2766283729999941, "nonpadding_tokens": 3964, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27372880100006114, "nonpadding_tokens": 15063, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27655862599999637, "nonpadding_tokens": 15291, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27704529600009664, "nonpadding_tokens": 17910, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003801280001880514, "prefetch": {"data_wait_seconds": 0.023353298000941436, "consumed_batches": 2491, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3041, "tokens": 220819971, "loss_tokens": 51405, "loss": 2.172854659079856, "lr": 0.0002989181234357202, "optimizer_lrs": [0.01992787489571468, 0.0002989181234357202], "gradient_norm": 0.781279981136322, "seconds": 0.2727755010000692, "nonpadding_tokens_per_second": 188451.67477114068, "ranks": [{"compute_seconds": 0.2696339309998166, "nonpadding_tokens": 6121, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2677462049998667, "nonpadding_tokens": 3785, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27017522000005556, "nonpadding_tokens": 4501, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27041483500011054, "nonpadding_tokens": 2675, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26607998500003305, "nonpadding_tokens": 4517, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.268456894999872, "nonpadding_tokens": 11445, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2657959470000151, "nonpadding_tokens": 13533, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2667917229998693, "nonpadding_tokens": 4828, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039233000006788643, "prefetch": {"data_wait_seconds": 0.02336148500099, "consumed_batches": 2492, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3042, "tokens": 220879575, "loss_tokens": 59604, "loss": 2.3682759336621704, "lr": 0.0002989174819914523, "optimizer_lrs": [0.019927832132763487, 0.0002989174819914523], "gradient_norm": 0.4194643795490265, "seconds": 0.3023589299998548, "nonpadding_tokens_per_second": 197129.94750983085, "ranks": [{"compute_seconds": 0.2999366999999893, "nonpadding_tokens": 8692, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2991402370000742, "nonpadding_tokens": 3161, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29623027000002367, "nonpadding_tokens": 2886, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2971169689999442, "nonpadding_tokens": 9349, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29494555099995523, "nonpadding_tokens": 13275, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2975067840000065, "nonpadding_tokens": 4563, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2989705120000963, "nonpadding_tokens": 11228, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2997965929998827, "nonpadding_tokens": 6450, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003721069999755855, "prefetch": {"data_wait_seconds": 0.02337049700095406, "consumed_batches": 2493, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3043, "tokens": 220939869, "loss_tokens": 60294, "loss": 2.204356206670647, "lr": 0.00029891683292893154, "optimizer_lrs": [0.01992778886192877, 0.00029891683292893154], "gradient_norm": 0.4081939458847046, "seconds": 0.2742127509998227, "nonpadding_tokens_per_second": 219880.36581143155, "ranks": [{"compute_seconds": 0.2660885520001557, "nonpadding_tokens": 1784, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2682534040000064, "nonpadding_tokens": 11745, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2710029059999215, "nonpadding_tokens": 4064, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27138468399994053, "nonpadding_tokens": 9420, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2718223909998869, "nonpadding_tokens": 8149, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2673434710000038, "nonpadding_tokens": 3652, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27145337299998573, "nonpadding_tokens": 10984, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.265119722999998, "nonpadding_tokens": 10496, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003472139999303181, "prefetch": {"data_wait_seconds": 0.023379359001140188, "consumed_batches": 2494, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3044, "tokens": 220996829, "loss_tokens": 56960, "loss": 2.47810607224368, "lr": 0.00029891621957881984, "optimizer_lrs": [0.019927747971921324, 0.00029891621957881984], "gradient_norm": 0.43550729751586914, "seconds": 0.2842700579999473, "nonpadding_tokens_per_second": 200372.84405102755, "ranks": [{"compute_seconds": 0.28198799800020424, "nonpadding_tokens": 6171, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2819029090001095, "nonpadding_tokens": 12122, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2787237610000375, "nonpadding_tokens": 3584, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28109445399991273, "nonpadding_tokens": 9361, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2817100269999173, "nonpadding_tokens": 4795, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27886092800008555, "nonpadding_tokens": 1561, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27746866799998315, "nonpadding_tokens": 11456, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2784299249999549, "nonpadding_tokens": 7910, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033268799984398356, "prefetch": {"data_wait_seconds": 0.023387756001241655, "consumed_batches": 2495, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3045, "tokens": 221057185, "loss_tokens": 60356, "loss": 2.5131291317350386, "lr": 0.0002989155694716139, "optimizer_lrs": [0.019927704631440925, 0.0002989155694716139], "gradient_norm": 0.40057116746902466, "seconds": 0.3217926360000547, "nonpadding_tokens_per_second": 187561.78124595038, "ranks": [{"compute_seconds": 0.31777811000006295, "nonpadding_tokens": 4137, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3141362559999834, "nonpadding_tokens": 12939, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31412212599980194, "nonpadding_tokens": 10134, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31493422699986695, "nonpadding_tokens": 11570, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31340983499990216, "nonpadding_tokens": 2363, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31575222300011774, "nonpadding_tokens": 11393, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.318753574000084, "nonpadding_tokens": 5179, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3192894060000526, "nonpadding_tokens": 2641, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003638799998952891, "prefetch": {"data_wait_seconds": 0.02339661200126386, "consumed_batches": 2496, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3046, "tokens": 221130545, "loss_tokens": 73360, "loss": 2.5480651922028352, "lr": 0.0002989147790341937, "optimizer_lrs": [0.019927651935612915, 0.0002989147790341937], "gradient_norm": 0.4034370481967926, "seconds": 0.2811702190001597, "nonpadding_tokens_per_second": 260909.56667056668, "ranks": [{"compute_seconds": 0.2775903310000558, "nonpadding_tokens": 2478, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2764950610001051, "nonpadding_tokens": 14542, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2740523969998776, "nonpadding_tokens": 9794, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27876514600006885, "nonpadding_tokens": 7452, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27331797300007565, "nonpadding_tokens": 18309, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2756033309999566, "nonpadding_tokens": 6006, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2764817629999925, "nonpadding_tokens": 7184, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2781180760000552, "nonpadding_tokens": 7595, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037296499999683874, "prefetch": {"data_wait_seconds": 0.02340552800110629, "consumed_batches": 2497, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3047, "tokens": 221193888, "loss_tokens": 63343, "loss": 2.444328645233096, "lr": 0.00029891409629694807, "optimizer_lrs": [0.01992760641979654, 0.00029891409629694807], "gradient_norm": 0.4068293571472168, "seconds": 0.29077764499993464, "nonpadding_tokens_per_second": 217839.99247952586, "ranks": [{"compute_seconds": 0.2851864269998714, "nonpadding_tokens": 4708, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28721908500006066, "nonpadding_tokens": 9675, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28631890699989526, "nonpadding_tokens": 16993, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2839182870000059, "nonpadding_tokens": 5819, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28751012299994727, "nonpadding_tokens": 6046, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2844040420000056, "nonpadding_tokens": 5087, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2882518720000462, "nonpadding_tokens": 8022, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28360965699994267, "nonpadding_tokens": 6993, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003581080000003567, "prefetch": {"data_wait_seconds": 0.023414619000917014, "consumed_batches": 2498, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3048, "tokens": 221257909, "loss_tokens": 64021, "loss": 2.1825187243248307, "lr": 0.0002989134060346626, "optimizer_lrs": [0.01992756040231084, 0.0002989134060346626], "gradient_norm": 0.37974151968955994, "seconds": 0.322611292999909, "nonpadding_tokens_per_second": 198446.245959586, "ranks": [{"compute_seconds": 0.3161007260000588, "nonpadding_tokens": 16820, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3172123740000643, "nonpadding_tokens": 8623, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3189372869999261, "nonpadding_tokens": 3946, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31978719300013836, "nonpadding_tokens": 4254, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.319785894000006, "nonpadding_tokens": 8227, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31709136599988597, "nonpadding_tokens": 5046, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31304923099992266, "nonpadding_tokens": 13183, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3201670930000091, "nonpadding_tokens": 3922, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004036970001379814, "prefetch": {"data_wait_seconds": 0.02342319000099735, "consumed_batches": 2499, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3049, "tokens": 221339642, "loss_tokens": 81733, "loss": 2.262971925354508, "lr": 0.0002989125244875173, "optimizer_lrs": [0.019927501632501156, 0.0002989125244875173], "gradient_norm": 0.42400985956192017, "seconds": 0.3645753609998792, "nonpadding_tokens_per_second": 224186.84514455457, "ranks": [{"compute_seconds": 0.36011663699991914, "nonpadding_tokens": 23222, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3563329440000871, "nonpadding_tokens": 7090, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35874433100002534, "nonpadding_tokens": 17836, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.359626968999919, "nonpadding_tokens": 11457, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3619054729999789, "nonpadding_tokens": 8293, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35513721000006626, "nonpadding_tokens": 5831, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3523320589999912, "nonpadding_tokens": 6461, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.36018052700001135, "nonpadding_tokens": 1543, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000379018000103315, "prefetch": {"data_wait_seconds": 0.023431895001024827, "consumed_batches": 2500, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3050, "tokens": 221385381, "loss_tokens": 45739, "loss": 2.327792782144341, "lr": 0.00029891203100529135, "optimizer_lrs": [0.01992746873368609, 0.00029891203100529135], "gradient_norm": 0.457358717918396, "seconds": 0.21030467099990346, "nonpadding_tokens_per_second": 217489.22542961966, "ranks": [{"compute_seconds": 0.17618022500005281, "nonpadding_tokens": 4060, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.17742295900006866, "nonpadding_tokens": 9271, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.1752835249999407, "nonpadding_tokens": 11389, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.1758003129998542, "nonpadding_tokens": 4048, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.1813857919999009, "nonpadding_tokens": 4075, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.17675927400000546, "nonpadding_tokens": 4072, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.18085175700002765, "nonpadding_tokens": 4897, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.18189464699980817, "nonpadding_tokens": 3927, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.013965108999855147, "prefetch": {"data_wait_seconds": 0.023442320001052508, "consumed_batches": 2501, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3051, "tokens": 221459844, "loss_tokens": 74463, "loss": 2.273099979184293, "lr": 0.00029891122737872334, "optimizer_lrs": [0.019927415158581558, 0.00029891122737872334], "gradient_norm": 0.46182331442832947, "seconds": 0.3959700180000709, "nonpadding_tokens_per_second": 188052.11661249227, "ranks": [{"compute_seconds": 0.3916813569999249, "nonpadding_tokens": 3627, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3901800270000422, "nonpadding_tokens": 17558, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3910519730000033, "nonpadding_tokens": 14206, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3877649039998232, "nonpadding_tokens": 6518, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.39329457799999545, "nonpadding_tokens": 12952, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3887277929998163, "nonpadding_tokens": 4072, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3859827270000551, "nonpadding_tokens": 11414, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3922463260000768, "nonpadding_tokens": 4116, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038495600006172026, "prefetch": {"data_wait_seconds": 0.023452614001101324, "consumed_batches": 2502, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3052, "tokens": 221507776, "loss_tokens": 47932, "loss": 2.1685476951723275, "lr": 0.0002989107099261792, "optimizer_lrs": [0.01992738066174528, 0.0002989107099261792], "gradient_norm": 0.4466554522514343, "seconds": 0.39510243899985653, "nonpadding_tokens_per_second": 121315.37360622926, "ranks": [{"compute_seconds": 0.3925918190000175, "nonpadding_tokens": 15001, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.38926927300008174, "nonpadding_tokens": 3609, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.38681748999988486, "nonpadding_tokens": 7576, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3874861100000544, "nonpadding_tokens": 4668, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.385932308999827, "nonpadding_tokens": 2003, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.389755986000182, "nonpadding_tokens": 3814, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3920081700000537, "nonpadding_tokens": 8616, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3862980769999922, "nonpadding_tokens": 2645, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004223400001137634, "prefetch": {"data_wait_seconds": 0.023462624001197696, "consumed_batches": 2503, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3053, "tokens": 221560760, "loss_tokens": 52984, "loss": 1.9764333336478936, "lr": 0.00029891013779201846, "optimizer_lrs": [0.0199273425194679, 0.00029891013779201846], "gradient_norm": 0.4139879047870636, "seconds": 0.3416742349998003, "nonpadding_tokens_per_second": 155071.68692433296, "ranks": [{"compute_seconds": 0.33308216000000357, "nonpadding_tokens": 12872, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33535909499983063, "nonpadding_tokens": 3150, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3314993060000688, "nonpadding_tokens": 4132, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3379149840000082, "nonpadding_tokens": 2000, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33861527899989596, "nonpadding_tokens": 5809, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33709006000003683, "nonpadding_tokens": 3908, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33811459399998967, "nonpadding_tokens": 6584, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33922103500003686, "nonpadding_tokens": 14529, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004262169998128229, "prefetch": {"data_wait_seconds": 0.023472376001294037, "consumed_batches": 2504, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3054, "tokens": 221621024, "loss_tokens": 60264, "loss": 2.5015992134607727, "lr": 0.0002989094868647915, "optimizer_lrs": [0.019927299124319436, 0.0002989094868647915], "gradient_norm": 0.42998096346855164, "seconds": 0.27287526800000705, "nonpadding_tokens_per_second": 220848.1568948874, "ranks": [{"compute_seconds": 0.2697529760000634, "nonpadding_tokens": 12427, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26593221500002073, "nonpadding_tokens": 5281, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2703746790000423, "nonpadding_tokens": 4785, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26884097099991777, "nonpadding_tokens": 3255, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26226893900002324, "nonpadding_tokens": 6346, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26651242200000524, "nonpadding_tokens": 8774, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2690164089999598, "nonpadding_tokens": 5012, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2629364289998648, "nonpadding_tokens": 14384, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040631399997437256, "prefetch": {"data_wait_seconds": 0.023481277001295098, "consumed_batches": 2505, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3055, "tokens": 221725806, "loss_tokens": 104782, "loss": 2.220995883119238, "lr": 0.000298908354626201, "optimizer_lrs": [0.019927223641746733, 0.000298908354626201], "gradient_norm": 0.40660732984542847, "seconds": 0.2933573410000463, "nonpadding_tokens_per_second": 357182.1303083855, "ranks": [{"compute_seconds": 0.2905803119999746, "nonpadding_tokens": 9820, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28663302099994326, "nonpadding_tokens": 5812, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28916062500002226, "nonpadding_tokens": 11670, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2880302149999352, "nonpadding_tokens": 12368, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2903316269998868, "nonpadding_tokens": 14053, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28915400599998975, "nonpadding_tokens": 11304, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2898352279999017, "nonpadding_tokens": 23074, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29084090499986814, "nonpadding_tokens": 16681, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00048820599999999104, "prefetch": {"data_wait_seconds": 0.023490269001470665, "consumed_batches": 2506, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3056, "tokens": 221827569, "loss_tokens": 101763, "loss": 2.2779962695183906, "lr": 0.00029890725444985024, "optimizer_lrs": [0.019927150296656683, 0.00029890725444985024], "gradient_norm": 0.44335877895355225, "seconds": 0.30335887399996864, "nonpadding_tokens_per_second": 335454.17234114115, "ranks": [{"compute_seconds": 0.301066880999997, "nonpadding_tokens": 23427, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30085764700015716, "nonpadding_tokens": 8531, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.299530422999851, "nonpadding_tokens": 23626, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3005067589999726, "nonpadding_tokens": 10396, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2966347039998709, "nonpadding_tokens": 5630, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2991936020000594, "nonpadding_tokens": 2134, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2968463309998697, "nonpadding_tokens": 25199, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29682888899992577, "nonpadding_tokens": 2820, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000351129999899058, "prefetch": {"data_wait_seconds": 0.02349866000140821, "consumed_batches": 2507, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3057, "tokens": 221902275, "loss_tokens": 74706, "loss": 2.397330870345086, "lr": 0.00029890644643986033, "optimizer_lrs": [0.019927096429324023, 0.00029890644643986033], "gradient_norm": 0.5026780962944031, "seconds": 0.2627595339999971, "nonpadding_tokens_per_second": 284313.1849975074, "ranks": [{"compute_seconds": 0.26019990500003587, "nonpadding_tokens": 12181, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2598388110000087, "nonpadding_tokens": 16845, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2569498929999554, "nonpadding_tokens": 2956, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25712730500004, "nonpadding_tokens": 6730, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25640866100002313, "nonpadding_tokens": 14146, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25736229800008914, "nonpadding_tokens": 9414, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2598424390000673, "nonpadding_tokens": 2895, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2604745380001532, "nonpadding_tokens": 9539, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003498629998830438, "prefetch": {"data_wait_seconds": 0.023507694001366986, "consumed_batches": 2508, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3058, "tokens": 221952634, "loss_tokens": 50359, "loss": 2.438570439246212, "lr": 0.00029890590159591574, "optimizer_lrs": [0.019927060106394383, 0.00029890590159591574], "gradient_norm": 0.6368219256401062, "seconds": 0.27744832399980623, "nonpadding_tokens_per_second": 181507.67420038613, "ranks": [{"compute_seconds": 0.27047708600002807, "nonpadding_tokens": 4179, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27108203199986747, "nonpadding_tokens": 5326, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.273597996000035, "nonpadding_tokens": 5279, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26836895199994615, "nonpadding_tokens": 2583, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2740184040001168, "nonpadding_tokens": 4669, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26916752099987207, "nonpadding_tokens": 14313, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2667465419999644, "nonpadding_tokens": 8916, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2749246729999868, "nonpadding_tokens": 5094, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000519928999892727, "prefetch": {"data_wait_seconds": 0.023516784001415658, "consumed_batches": 2509, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3059, "tokens": 222016121, "loss_tokens": 63487, "loss": 2.2689790331091406, "lr": 0.0002989052145249712, "optimizer_lrs": [0.01992701430166475, 0.0002989052145249712], "gradient_norm": 0.4945368468761444, "seconds": 0.30597877199988943, "nonpadding_tokens_per_second": 207488.25019803314, "ranks": [{"compute_seconds": 0.3019663469999614, "nonpadding_tokens": 2383, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30360596500008796, "nonpadding_tokens": 14425, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30021699000008084, "nonpadding_tokens": 12726, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3013119409999945, "nonpadding_tokens": 4686, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29942606999998134, "nonpadding_tokens": 7065, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.300731007999957, "nonpadding_tokens": 12840, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2995310790001895, "nonpadding_tokens": 3918, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3035537490000024, "nonpadding_tokens": 5444, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040858400006982265, "prefetch": {"data_wait_seconds": 0.023529722001470077, "consumed_batches": 2510, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3060, "tokens": 222066005, "loss_tokens": 49884, "loss": 2.1183548958583915, "lr": 0.00029890467451814187, "optimizer_lrs": [0.019926978301209458, 0.00029890467451814187], "gradient_norm": 0.48188546299934387, "seconds": 0.2709137470001224, "nonpadding_tokens_per_second": 184132.4058021222, "ranks": [{"compute_seconds": 0.26433586200005266, "nonpadding_tokens": 4053, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2654137009999431, "nonpadding_tokens": 2003, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26771402500003205, "nonpadding_tokens": 8656, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26830435700003363, "nonpadding_tokens": 7835, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2685782430000927, "nonpadding_tokens": 2049, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2661635929998738, "nonpadding_tokens": 4034, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.262652929000069, "nonpadding_tokens": 12576, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2632112100000086, "nonpadding_tokens": 8678, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003515719999995781, "prefetch": {"data_wait_seconds": 0.023538407001481687, "consumed_batches": 2511, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3061, "tokens": 222104023, "loss_tokens": 38018, "loss": 2.118704462359935, "lr": 0.00029890426287470086, "optimizer_lrs": [0.01992695085831339, 0.00029890426287470086], "gradient_norm": 0.44623133540153503, "seconds": 0.24862792600015382, "nonpadding_tokens_per_second": 152911.22204822832, "ranks": [{"compute_seconds": 0.24119700999995075, "nonpadding_tokens": 6124, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24245727500010616, "nonpadding_tokens": 2604, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24533596200012653, "nonpadding_tokens": 6624, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24008061900008215, "nonpadding_tokens": 6838, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24629460899996047, "nonpadding_tokens": 3443, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24378788999979406, "nonpadding_tokens": 4050, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23769273599987173, "nonpadding_tokens": 2965, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23848755499989238, "nonpadding_tokens": 5370, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003486959999463579, "prefetch": {"data_wait_seconds": 0.023546542001398052, "consumed_batches": 2512, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3062, "tokens": 222182530, "loss_tokens": 78507, "loss": 2.3252580183932645, "lr": 0.0002989034125890927, "optimizer_lrs": [0.019926894172606183, 0.0002989034125890927], "gradient_norm": 0.47346818447113037, "seconds": 0.3431462469998223, "nonpadding_tokens_per_second": 228785.83311459224, "ranks": [{"compute_seconds": 0.3395983579998756, "nonpadding_tokens": 3136, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3389882189999298, "nonpadding_tokens": 6449, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3391999250000026, "nonpadding_tokens": 12288, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33616356500010625, "nonpadding_tokens": 10188, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3352495019998969, "nonpadding_tokens": 13778, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3375509890001922, "nonpadding_tokens": 17281, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.334859508999898, "nonpadding_tokens": 8394, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3408181309998781, "nonpadding_tokens": 6993, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003936620000786206, "prefetch": {"data_wait_seconds": 0.023554649001383154, "consumed_batches": 2513, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3063, "tokens": 222260916, "loss_tokens": 78386, "loss": 2.530112998494629, "lr": 0.00029890256328632295, "optimizer_lrs": [0.01992683755242153, 0.00029890256328632295], "gradient_norm": 0.4419746398925781, "seconds": 0.3751372449999053, "nonpadding_tokens_per_second": 208952.85937289376, "ranks": [{"compute_seconds": 0.37123242300003767, "nonpadding_tokens": 6695, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3704025790000287, "nonpadding_tokens": 3465, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37158718300020155, "nonpadding_tokens": 7982, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.368449450000071, "nonpadding_tokens": 13540, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3727689830000145, "nonpadding_tokens": 15577, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36991188499996497, "nonpadding_tokens": 1282, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.37246378800000457, "nonpadding_tokens": 14707, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3677987349999512, "nonpadding_tokens": 15138, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037067400012347207, "prefetch": {"data_wait_seconds": 0.023563697001421247, "consumed_batches": 2514, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3064, "tokens": 222325316, "loss_tokens": 64400, "loss": 2.2631431968167703, "lr": 0.0002989018652751578, "optimizer_lrs": [0.019926791018343853, 0.0002989018652751578], "gradient_norm": 0.4557761251926422, "seconds": 0.3441690509998807, "nonpadding_tokens_per_second": 187117.34774787267, "ranks": [{"compute_seconds": 0.33977237800013427, "nonpadding_tokens": 9714, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33853074800003924, "nonpadding_tokens": 4449, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33541110900000604, "nonpadding_tokens": 2419, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34093230099983884, "nonpadding_tokens": 6602, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.34137521599996035, "nonpadding_tokens": 11118, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3385175780001646, "nonpadding_tokens": 18638, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34102448200019353, "nonpadding_tokens": 6891, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34164725999994516, "nonpadding_tokens": 4569, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003756969999813009, "prefetch": {"data_wait_seconds": 0.023573514001284934, "consumed_batches": 2515, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3065, "tokens": 222388792, "loss_tokens": 63476, "loss": 2.578479956991619, "lr": 0.00029890117706266554, "optimizer_lrs": [0.019926745137511036, 0.00029890117706266554], "gradient_norm": 0.438099205493927, "seconds": 0.25664973500011, "nonpadding_tokens_per_second": 247325.40635575872, "ranks": [{"compute_seconds": 0.25022206100015865, "nonpadding_tokens": 7659, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2501471160001074, "nonpadding_tokens": 2626, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25266964599995845, "nonpadding_tokens": 9025, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24931497399984437, "nonpadding_tokens": 2086, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25382235500001116, "nonpadding_tokens": 6610, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25000910099993234, "nonpadding_tokens": 5327, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2478116280001359, "nonpadding_tokens": 18876, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2540737760000411, "nonpadding_tokens": 11267, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041899000007106224, "prefetch": {"data_wait_seconds": 0.023583252001074584, "consumed_batches": 2516, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3066, "tokens": 222448222, "loss_tokens": 59430, "loss": 2.196400965421504, "lr": 0.0002989005325226544, "optimizer_lrs": [0.01992670216817696, 0.0002989005325226544], "gradient_norm": 0.5037059187889099, "seconds": 0.2960883950001971, "nonpadding_tokens_per_second": 200717.08653073158, "ranks": [{"compute_seconds": 0.2874849600000289, "nonpadding_tokens": 9866, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28954472599980363, "nonpadding_tokens": 3130, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2869672780000201, "nonpadding_tokens": 3369, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28720504299985805, "nonpadding_tokens": 18702, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29328154399991035, "nonpadding_tokens": 8643, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29046214999993936, "nonpadding_tokens": 9836, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2857409840000855, "nonpadding_tokens": 3772, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2932144789999711, "nonpadding_tokens": 2112, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034485800006223144, "prefetch": {"data_wait_seconds": 0.023591684001033286, "consumed_batches": 2517, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3067, "tokens": 222554272, "loss_tokens": 106050, "loss": 2.316713077557756, "lr": 0.0002988993819041787, "optimizer_lrs": [0.01992662546027858, 0.0002988993819041787], "gradient_norm": 0.4176940619945526, "seconds": 0.28786723099983647, "nonpadding_tokens_per_second": 368398.999884986, "ranks": [{"compute_seconds": 0.28424285200003396, "nonpadding_tokens": 7520, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2825659650000034, "nonpadding_tokens": 19235, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2835872759999347, "nonpadding_tokens": 9858, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28446924000013496, "nonpadding_tokens": 11882, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2785792829999991, "nonpadding_tokens": 10896, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.281206893999979, "nonpadding_tokens": 17228, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2783569850000731, "nonpadding_tokens": 9709, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28526729000009254, "nonpadding_tokens": 19722, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004169759999967937, "prefetch": {"data_wait_seconds": 0.023601105000992106, "consumed_batches": 2518, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3068, "tokens": 222627691, "loss_tokens": 73419, "loss": 2.2894088638499572, "lr": 0.0002988985849735955, "optimizer_lrs": [0.019926572331573036, 0.0002988985849735955], "gradient_norm": 0.42150288820266724, "seconds": 0.33951633400010905, "nonpadding_tokens_per_second": 216245.8551993449, "ranks": [{"compute_seconds": 0.33635615599996527, "nonpadding_tokens": 9944, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3346066490000794, "nonpadding_tokens": 3343, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3364390229999117, "nonpadding_tokens": 12106, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3317767519999961, "nonpadding_tokens": 4936, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3371390209999845, "nonpadding_tokens": 6285, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33421278800005894, "nonpadding_tokens": 6229, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3367666729998291, "nonpadding_tokens": 8383, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33111996499997076, "nonpadding_tokens": 22193, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003712819998327177, "prefetch": {"data_wait_seconds": 0.023609362000982514, "consumed_batches": 2519, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3069, "tokens": 222702622, "loss_tokens": 74931, "loss": 2.2558305724599967, "lr": 0.00029889777133475833, "optimizer_lrs": [0.01992651808898389, 0.00029889777133475833], "gradient_norm": 0.6585734486579895, "seconds": 0.36060120799993456, "nonpadding_tokens_per_second": 207794.64499190918, "ranks": [{"compute_seconds": 0.3580251920000137, "nonpadding_tokens": 17393, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35642641399999775, "nonpadding_tokens": 7953, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3532032900000104, "nonpadding_tokens": 5858, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35744088100000226, "nonpadding_tokens": 3401, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3577780220000477, "nonpadding_tokens": 20229, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35471583100002135, "nonpadding_tokens": 10583, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.351931766000007, "nonpadding_tokens": 5197, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35809489600001143, "nonpadding_tokens": 4317, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038264799991338805, "prefetch": {"data_wait_seconds": 0.023619124000973102, "consumed_batches": 2520, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3070, "tokens": 222764774, "loss_tokens": 62152, "loss": 2.506118316787875, "lr": 0.00029889709622979183, "optimizer_lrs": [0.019926473081986123, 0.00029889709622979183], "gradient_norm": 0.5061163306236267, "seconds": 0.36309398800017334, "nonpadding_tokens_per_second": 171173.3106414594, "ranks": [{"compute_seconds": 0.35953493900001376, "nonpadding_tokens": 24104, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35675556899991534, "nonpadding_tokens": 4464, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35821902899988345, "nonpadding_tokens": 4603, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35998011899982885, "nonpadding_tokens": 6359, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3572075780000432, "nonpadding_tokens": 5196, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36068360500007657, "nonpadding_tokens": 2934, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3571224300001177, "nonpadding_tokens": 9616, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3580909099998735, "nonpadding_tokens": 4876, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037434499995470105, "prefetch": {"data_wait_seconds": 0.023627092001063943, "consumed_batches": 2521, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3071, "tokens": 222827782, "loss_tokens": 63008, "loss": 2.80486828061516, "lr": 0.000298896411616708, "optimizer_lrs": [0.019926427441113868, 0.000298896411616708], "gradient_norm": 0.5971644520759583, "seconds": 0.35843605800005207, "nonpadding_tokens_per_second": 175785.8859166196, "ranks": [{"compute_seconds": 0.3543658300000061, "nonpadding_tokens": 5218, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3521630919999552, "nonpadding_tokens": 3533, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3509558199998537, "nonpadding_tokens": 6471, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3551780139998755, "nonpadding_tokens": 11759, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3484749799999918, "nonpadding_tokens": 18932, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3528161589999854, "nonpadding_tokens": 6780, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34872697100013283, "nonpadding_tokens": 8651, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35600312400015355, "nonpadding_tokens": 1664, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003759179999178741, "prefetch": {"data_wait_seconds": 0.023636014001070293, "consumed_batches": 2522, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3072, "tokens": 222895240, "loss_tokens": 67458, "loss": 2.139068244314981, "lr": 0.00029889567841769055, "optimizer_lrs": [0.019926378561179372, 0.00029889567841769055], "gradient_norm": 0.44288790225982666, "seconds": 0.28648855500000536, "nonpadding_tokens_per_second": 235464.90365033512, "ranks": [{"compute_seconds": 0.2831464270000197, "nonpadding_tokens": 7386, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2804125070001646, "nonpadding_tokens": 6958, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28201037000008, "nonpadding_tokens": 5804, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27852137300010327, "nonpadding_tokens": 5020, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2831406809998498, "nonpadding_tokens": 8257, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2801520050002182, "nonpadding_tokens": 10202, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27627641199978825, "nonpadding_tokens": 18518, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28388316599989594, "nonpadding_tokens": 5313, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003729269999439566, "prefetch": {"data_wait_seconds": 0.023645184000955055, "consumed_batches": 2523, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3073, "tokens": 222953969, "loss_tokens": 58729, "loss": 2.187473527771629, "lr": 0.00029889503989647277, "optimizer_lrs": [0.019926335993098187, 0.00029889503989647277], "gradient_norm": 0.42562419176101685, "seconds": 0.29368936099990606, "nonpadding_tokens_per_second": 199969.7905298612, "ranks": [{"compute_seconds": 0.289362452999967, "nonpadding_tokens": 2672, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28741883199995755, "nonpadding_tokens": 2548, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2862161649998143, "nonpadding_tokens": 6369, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28804544800004805, "nonpadding_tokens": 9255, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2851747469999282, "nonpadding_tokens": 14176, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28842891900012546, "nonpadding_tokens": 3934, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29054420000011305, "nonpadding_tokens": 16579, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29123466500004724, "nonpadding_tokens": 3196, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003770130001612415, "prefetch": {"data_wait_seconds": 0.02365354500098249, "consumed_batches": 2524, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3074, "tokens": 223003089, "loss_tokens": 49120, "loss": 2.3988984247760587, "lr": 0.0002988945057063584, "optimizer_lrs": [0.019926300380423895, 0.0002988945057063584], "gradient_norm": 0.43800801038742065, "seconds": 0.22969856999998228, "nonpadding_tokens_per_second": 213845.47583384515, "ranks": [{"compute_seconds": 0.22482831100001022, "nonpadding_tokens": 6600, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22333237400016515, "nonpadding_tokens": 9580, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22567568399995253, "nonpadding_tokens": 6544, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22219064999990223, "nonpadding_tokens": 5475, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22656818700011172, "nonpadding_tokens": 5275, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22336528999994698, "nonpadding_tokens": 6980, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22613536900007603, "nonpadding_tokens": 3579, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22706861599999684, "nonpadding_tokens": 5087, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000402861000111443, "prefetch": {"data_wait_seconds": 0.023661875000883015, "consumed_batches": 2525, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3075, "tokens": 223097137, "loss_tokens": 94048, "loss": 2.206980245459765, "lr": 0.00029889348255626193, "optimizer_lrs": [0.019926232170417463, 0.00029889348255626193], "gradient_norm": 0.43846800923347473, "seconds": 0.3627795149998292, "nonpadding_tokens_per_second": 259242.8627069648, "ranks": [{"compute_seconds": 0.35458985900004336, "nonpadding_tokens": 18495, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.355893579999929, "nonpadding_tokens": 24875, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3588175999998384, "nonpadding_tokens": 16383, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35376469300013014, "nonpadding_tokens": 5281, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3598108379999303, "nonpadding_tokens": 8490, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3571651419999853, "nonpadding_tokens": 4447, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3519639379999262, "nonpadding_tokens": 9448, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.360185969999975, "nonpadding_tokens": 6629, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045059299986860424, "prefetch": {"data_wait_seconds": 0.023669106000809848, "consumed_batches": 2526, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3076, "tokens": 223164576, "loss_tokens": 67439, "loss": 2.374005119441273, "lr": 0.00029889274859586735, "optimizer_lrs": [0.01992618323972449, 0.00029889274859586735], "gradient_norm": 0.5114300847053528, "seconds": 0.34120971500010455, "nonpadding_tokens_per_second": 197646.77567864483, "ranks": [{"compute_seconds": 0.2714256970000406, "nonpadding_tokens": 5189, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27446286100007455, "nonpadding_tokens": 4306, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27089924999995674, "nonpadding_tokens": 11576, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.276726139999937, "nonpadding_tokens": 14844, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2772549320000053, "nonpadding_tokens": 4642, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2734123590000763, "nonpadding_tokens": 7498, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27676461900000504, "nonpadding_tokens": 2831, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2778506100000868, "nonpadding_tokens": 16553, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.04189119600005142, "prefetch": {"data_wait_seconds": 0.023679502000689467, "consumed_batches": 2527, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3077, "tokens": 223223039, "loss_tokens": 58463, "loss": 2.5933592614132017, "lr": 0.00029889211212807466, "optimizer_lrs": [0.01992614080853831, 0.00029889211212807466], "gradient_norm": 0.46740421652793884, "seconds": 0.2962164279999797, "nonpadding_tokens_per_second": 197365.8260439357, "ranks": [{"compute_seconds": 0.2919256189998123, "nonpadding_tokens": 5721, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28969170499999564, "nonpadding_tokens": 10134, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29180271000018365, "nonpadding_tokens": 13428, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2923671700000341, "nonpadding_tokens": 3723, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2926231879998795, "nonpadding_tokens": 13490, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2889204170000994, "nonpadding_tokens": 2855, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29249512600017624, "nonpadding_tokens": 5834, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2936527169999863, "nonpadding_tokens": 3278, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003534590000526805, "prefetch": {"data_wait_seconds": 0.023688058000701062, "consumed_batches": 2528, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3078, "tokens": 223269997, "loss_tokens": 46958, "loss": 2.3488183589590697, "lr": 0.00029889160077961266, "optimizer_lrs": [0.019926106718640844, 0.00029889160077961266], "gradient_norm": 0.42930442094802856, "seconds": 0.3033972010000525, "nonpadding_tokens_per_second": 154774.0053145443, "ranks": [{"compute_seconds": 0.29641367099998206, "nonpadding_tokens": 2759, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29777862299988556, "nonpadding_tokens": 10723, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30022808000012446, "nonpadding_tokens": 10301, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3006633420000071, "nonpadding_tokens": 5650, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29438394299995707, "nonpadding_tokens": 6145, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2984292529999948, "nonpadding_tokens": 4116, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30083527200008575, "nonpadding_tokens": 3201, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29524997300018185, "nonpadding_tokens": 4063, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004332790001626563, "prefetch": {"data_wait_seconds": 0.023695953000654413, "consumed_batches": 2529, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3079, "tokens": 223330124, "loss_tokens": 60127, "loss": 2.2596680775691453, "lr": 0.0002988909458559767, "optimizer_lrs": [0.019926063057065117, 0.0002988909458559767], "gradient_norm": 0.42495104670524597, "seconds": 0.33475028599991674, "nonpadding_tokens_per_second": 179617.47163381052, "ranks": [{"compute_seconds": 0.328443494999874, "nonpadding_tokens": 4847, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32970685199984473, "nonpadding_tokens": 12480, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33060791099978815, "nonpadding_tokens": 3597, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.325733320000154, "nonpadding_tokens": 2612, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33126954500016836, "nonpadding_tokens": 6440, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32716985100000784, "nonpadding_tokens": 4014, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33135188299979745, "nonpadding_tokens": 14139, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.332217159000038, "nonpadding_tokens": 11998, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003590430001167988, "prefetch": {"data_wait_seconds": 0.023704786000465106, "consumed_batches": 2530, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3080, "tokens": 223420179, "loss_tokens": 90055, "loss": 2.1694292723890953, "lr": 0.00029888996458611535, "optimizer_lrs": [0.019925997639074357, 0.00029888996458611535], "gradient_norm": 0.3944982886314392, "seconds": 0.3106343659999311, "nonpadding_tokens_per_second": 289906.75165676925, "ranks": [{"compute_seconds": 0.30719930700001896, "nonpadding_tokens": 2530, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30491881100010687, "nonpadding_tokens": 5343, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3069351879998976, "nonpadding_tokens": 18508, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3024782850000065, "nonpadding_tokens": 9834, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30795758299996123, "nonpadding_tokens": 16106, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30365950100008376, "nonpadding_tokens": 24662, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30769301800000903, "nonpadding_tokens": 9367, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3014703000001191, "nonpadding_tokens": 3705, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046175600004971784, "prefetch": {"data_wait_seconds": 0.023715798000424115, "consumed_batches": 2531, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3081, "tokens": 223476329, "loss_tokens": 56150, "loss": 2.152741818788958, "lr": 0.0002988893525379362, "optimizer_lrs": [0.019925956835862416, 0.0002988893525379362], "gradient_norm": 0.3814922869205475, "seconds": 0.51266591600006, "nonpadding_tokens_per_second": 109525.51797883406, "ranks": [{"compute_seconds": 0.4676285909999933, "nonpadding_tokens": 6931, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.48753766200002246, "nonpadding_tokens": 10591, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.48708520700006375, "nonpadding_tokens": 3732, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.48780649600007564, "nonpadding_tokens": 8868, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.4825542860000951, "nonpadding_tokens": 4541, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4859961849999763, "nonpadding_tokens": 10281, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4825714660000813, "nonpadding_tokens": 2403, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.48877922099995885, "nonpadding_tokens": 8803, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.016869196000016018, "prefetch": {"data_wait_seconds": 0.023725076000530976, "consumed_batches": 2532, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3082, "tokens": 223541554, "loss_tokens": 65225, "loss": 2.079308882713683, "lr": 0.00029888864135921544, "optimizer_lrs": [0.0199259094239477, 0.00029888864135921544], "gradient_norm": 0.4494835138320923, "seconds": 0.26660802100013825, "nonpadding_tokens_per_second": 244647.55319558136, "ranks": [{"compute_seconds": 0.26059863799991945, "nonpadding_tokens": 8629, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2599904969999898, "nonpadding_tokens": 7194, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25702942399993844, "nonpadding_tokens": 7513, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2580789769999683, "nonpadding_tokens": 10793, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2638786819998131, "nonpadding_tokens": 11363, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2606839029999719, "nonpadding_tokens": 5144, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26324480900007075, "nonpadding_tokens": 5872, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2636898199998541, "nonpadding_tokens": 8717, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044776400000046124, "prefetch": {"data_wait_seconds": 0.02373409100046331, "consumed_batches": 2533, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3083, "tokens": 223607928, "loss_tokens": 66374, "loss": 2.344641020957001, "lr": 0.00029888791741970483, "optimizer_lrs": [0.019925861161313656, 0.00029888791741970483], "gradient_norm": 0.39941737055778503, "seconds": 0.3559881400001359, "nonpadding_tokens_per_second": 186450.0317341321, "ranks": [{"compute_seconds": 0.3494281250000313, "nonpadding_tokens": 15577, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35210462600002757, "nonpadding_tokens": 6640, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35203573600006166, "nonpadding_tokens": 5520, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35332990699998845, "nonpadding_tokens": 3047, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3478973960000076, "nonpadding_tokens": 3566, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34990144099992904, "nonpadding_tokens": 6130, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34836178099999415, "nonpadding_tokens": 17085, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3485131330000968, "nonpadding_tokens": 8809, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044544000002133544, "prefetch": {"data_wait_seconds": 0.023744553000554447, "consumed_batches": 2534, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3084, "tokens": 223682178, "loss_tokens": 74250, "loss": 2.396906355218855, "lr": 0.0002988871072986961, "optimizer_lrs": [0.01992580715324641, 0.0002988871072986961], "gradient_norm": 0.5500463247299194, "seconds": 0.37556112900006156, "nonpadding_tokens_per_second": 197704.16655656564, "ranks": [{"compute_seconds": 0.372141640999871, "nonpadding_tokens": 16968, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36844613099992785, "nonpadding_tokens": 9124, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37114547400005904, "nonpadding_tokens": 16392, "peak_allocated_bytes": 14999285248, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3718061319998469, "nonpadding_tokens": 10738, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3731436489999851, "nonpadding_tokens": 7965, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37052191799989487, "nonpadding_tokens": 5196, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3680727989999468, "nonpadding_tokens": 4333, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3683083290000013, "nonpadding_tokens": 3534, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036300000010669464, "prefetch": {"data_wait_seconds": 0.0237534190004709, "consumed_batches": 2535, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3085, "tokens": 223815177, "loss_tokens": 132999, "loss": 2.1194672986263057, "lr": 0.00029888565544943175, "optimizer_lrs": [0.019925710363295452, 0.00029888565544943175], "gradient_norm": 0.45111241936683655, "seconds": 0.29469039699984023, "nonpadding_tokens_per_second": 451317.7265157782, "ranks": [{"compute_seconds": 0.28256389700004547, "nonpadding_tokens": 6443, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28909305599995605, "nonpadding_tokens": 10769, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28678679199992985, "nonpadding_tokens": 23616, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29105891299991526, "nonpadding_tokens": 6395, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2855600450000111, "nonpadding_tokens": 29399, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28946341899995787, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29102203899992674, "nonpadding_tokens": 6239, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28625871799999913, "nonpadding_tokens": 17370, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004101440001704759, "prefetch": {"data_wait_seconds": 0.023764338000503926, "consumed_batches": 2536, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3086, "tokens": 223855684, "loss_tokens": 40507, "loss": 2.4736667813587774, "lr": 0.00029888521307798205, "optimizer_lrs": [0.01992568087186547, 0.00029888521307798205], "gradient_norm": 0.4555966556072235, "seconds": 0.3278026860000409, "nonpadding_tokens_per_second": 123571.2876373287, "ranks": [{"compute_seconds": 0.3203661899999588, "nonpadding_tokens": 2001, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32073766300004536, "nonpadding_tokens": 4266, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32107500600000094, "nonpadding_tokens": 3749, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32509682099998827, "nonpadding_tokens": 6362, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32456861900004697, "nonpadding_tokens": 2682, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3224537839998902, "nonpadding_tokens": 5079, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31871813599991583, "nonpadding_tokens": 2895, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32520898199982184, "nonpadding_tokens": 13473, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043658599997797864, "prefetch": {"data_wait_seconds": 0.023789973000660325, "consumed_batches": 2537, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3087, "tokens": 223938844, "loss_tokens": 83160, "loss": 2.1098924512987014, "lr": 0.00029888430462493267, "optimizer_lrs": [0.019925620308328847, 0.00029888430462493267], "gradient_norm": 0.3985576331615448, "seconds": 0.3049061399999573, "nonpadding_tokens_per_second": 272739.66998503753, "ranks": [{"compute_seconds": 0.30069507400003204, "nonpadding_tokens": 18655, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29906284899993807, "nonpadding_tokens": 11282, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29606245199988734, "nonpadding_tokens": 8603, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3023622589998922, "nonpadding_tokens": 6263, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2947985600001175, "nonpadding_tokens": 8116, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30047526599992125, "nonpadding_tokens": 13205, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2952187169998979, "nonpadding_tokens": 8775, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29546774000004916, "nonpadding_tokens": 8261, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037589899989143305, "prefetch": {"data_wait_seconds": 0.0237985710006825, "consumed_batches": 2538, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3088, "tokens": 224028986, "loss_tokens": 90142, "loss": 2.253773040036831, "lr": 0.0002988833194832356, "optimizer_lrs": [0.01992555463221571, 0.0002988833194832356], "gradient_norm": 0.4095892608165741, "seconds": 0.26239752200012845, "nonpadding_tokens_per_second": 343532.20759438374, "ranks": [{"compute_seconds": 0.2562375069999234, "nonpadding_tokens": 4339, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2566385939999236, "nonpadding_tokens": 3540, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.259862935000001, "nonpadding_tokens": 7042, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25413200599996344, "nonpadding_tokens": 4593, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2524486539998634, "nonpadding_tokens": 11606, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2575443179998729, "nonpadding_tokens": 14244, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25283914900001037, "nonpadding_tokens": 23726, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.252772923999828, "nonpadding_tokens": 21052, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003441189999193739, "prefetch": {"data_wait_seconds": 0.02380789600051969, "consumed_batches": 2539, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3089, "tokens": 224081584, "loss_tokens": 52598, "loss": 2.4287315463515724, "lr": 0.00029888274445148195, "optimizer_lrs": [0.019925516296765464, 0.00029888274445148195], "gradient_norm": 0.4217221140861511, "seconds": 0.2662488899998152, "nonpadding_tokens_per_second": 197551.99730611648, "ranks": [{"compute_seconds": 0.2621983939998245, "nonpadding_tokens": 4912, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2589992200000779, "nonpadding_tokens": 10480, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25643398400006845, "nonpadding_tokens": 4119, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2632005620000655, "nonpadding_tokens": 5014, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2637157459998889, "nonpadding_tokens": 3217, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2605773620000491, "nonpadding_tokens": 4072, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2634778120000192, "nonpadding_tokens": 7002, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25612137600001006, "nonpadding_tokens": 13782, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000433054000041011, "prefetch": {"data_wait_seconds": 0.02381747200043094, "consumed_batches": 2540, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3090, "tokens": 224150174, "loss_tokens": 68590, "loss": 2.16478509075667, "lr": 0.0002988819943644962, "optimizer_lrs": [0.019925466290966413, 0.0002988819943644962], "gradient_norm": 0.5717185735702515, "seconds": 0.36903032100008204, "nonpadding_tokens_per_second": 185865.4861045544, "ranks": [{"compute_seconds": 0.36232155900006546, "nonpadding_tokens": 7719, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3631792660000883, "nonpadding_tokens": 9228, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35992412299992793, "nonpadding_tokens": 7055, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3651530699999057, "nonpadding_tokens": 12197, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3657938940000349, "nonpadding_tokens": 5984, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3629248180000104, "nonpadding_tokens": 3882, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.366367099999934, "nonpadding_tokens": 8035, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3598506920000091, "nonpadding_tokens": 14490, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003973340001266479, "prefetch": {"data_wait_seconds": 0.02382708600039507, "consumed_batches": 2541, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3091, "tokens": 224207844, "loss_tokens": 57670, "loss": 2.4032458383908444, "lr": 0.0002988813635025438, "optimizer_lrs": [0.019925424233502922, 0.0002988813635025438], "gradient_norm": 0.5604372620582581, "seconds": 0.3570811329998378, "nonpadding_tokens_per_second": 161503.9123336326, "ranks": [{"compute_seconds": 0.35326141600012306, "nonpadding_tokens": 2552, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35161846199980573, "nonpadding_tokens": 1725, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3534732420000637, "nonpadding_tokens": 4227, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34996140699990974, "nonpadding_tokens": 16451, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3483212060000369, "nonpadding_tokens": 12114, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3512266239999917, "nonpadding_tokens": 12883, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3479403329999968, "nonpadding_tokens": 4625, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35475606099998913, "nonpadding_tokens": 3093, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038170800007719663, "prefetch": {"data_wait_seconds": 0.02383611500040672, "consumed_batches": 2542, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3092, "tokens": 224251584, "loss_tokens": 43740, "loss": 2.3120890132030176, "lr": 0.0002988808849050684, "optimizer_lrs": [0.019925392327004562, 0.0002988808849050684], "gradient_norm": 0.40902408957481384, "seconds": 0.2256739260001268, "nonpadding_tokens_per_second": 193819.46676451855, "ranks": [{"compute_seconds": 0.22253476900004898, "nonpadding_tokens": 3915, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21868539899992356, "nonpadding_tokens": 7834, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21670699499986767, "nonpadding_tokens": 8731, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22248369499993714, "nonpadding_tokens": 7558, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22285725799997635, "nonpadding_tokens": 4645, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21984557999985554, "nonpadding_tokens": 5641, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22248398300007466, "nonpadding_tokens": 3907, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22319032500013236, "nonpadding_tokens": 1509, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003810190000876901, "prefetch": {"data_wait_seconds": 0.023846150000466082, "consumed_batches": 2543, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3093, "tokens": 224297406, "loss_tokens": 45822, "loss": 2.221813314292261, "lr": 0.00029888038341728876, "optimizer_lrs": [0.01992535889448592, 0.00029888038341728876], "gradient_norm": 0.44668442010879517, "seconds": 0.24247567699990213, "nonpadding_tokens_per_second": 188975.65548406943, "ranks": [{"compute_seconds": 0.23820034599998507, "nonpadding_tokens": 3692, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23598199100001693, "nonpadding_tokens": 2327, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23388411299993095, "nonpadding_tokens": 12073, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23463164999998298, "nonpadding_tokens": 6244, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2333021209999515, "nonpadding_tokens": 4881, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2360814369999389, "nonpadding_tokens": 7238, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23273117799999454, "nonpadding_tokens": 4040, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2400489760000255, "nonpadding_tokens": 5327, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003446829998665635, "prefetch": {"data_wait_seconds": 0.02385572900057298, "consumed_batches": 2544, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3094, "tokens": 224354928, "loss_tokens": 57522, "loss": 2.2145357156826955, "lr": 0.0002988797537233326, "optimizer_lrs": [0.01992531691488884, 0.0002988797537233326], "gradient_norm": 0.39344099164009094, "seconds": 0.28399170300008336, "nonpadding_tokens_per_second": 202548.17092308897, "ranks": [{"compute_seconds": 0.28054998299990075, "nonpadding_tokens": 7409, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2785353499998564, "nonpadding_tokens": 11713, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28113409499997033, "nonpadding_tokens": 6857, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2815851620000558, "nonpadding_tokens": 3681, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27550649099998736, "nonpadding_tokens": 3485, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2792747539999709, "nonpadding_tokens": 6151, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2755803439999909, "nonpadding_tokens": 2230, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2761498269999265, "nonpadding_tokens": 15996, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003836759999558126, "prefetch": {"data_wait_seconds": 0.023865260000732746, "consumed_batches": 2545, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3095, "tokens": 224441010, "loss_tokens": 86082, "loss": 2.2488704447503545, "lr": 0.0002988788110534379, "optimizer_lrs": [0.019925254070229194, 0.0002988788110534379], "gradient_norm": 0.423505574464798, "seconds": 0.3574732249999215, "nonpadding_tokens_per_second": 240806.84644288785, "ranks": [{"compute_seconds": 0.323649037999985, "nonpadding_tokens": 14138, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34424401399996896, "nonpadding_tokens": 11708, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34114048499986893, "nonpadding_tokens": 5520, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34207329199989545, "nonpadding_tokens": 9082, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3400726319998739, "nonpadding_tokens": 5074, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34313526300002195, "nonpadding_tokens": 5680, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34766678500000125, "nonpadding_tokens": 27580, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3405476739999358, "nonpadding_tokens": 7300, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0018616310001107195, "prefetch": {"data_wait_seconds": 0.02387287900064905, "consumed_batches": 2546, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3096, "tokens": 224483516, "loss_tokens": 42506, "loss": 2.1979661694819557, "lr": 0.00029887834543158365, "optimizer_lrs": [0.019925223028772244, 0.00029887834543158365], "gradient_norm": 0.4131394922733307, "seconds": 0.32371416499995576, "nonpadding_tokens_per_second": 131307.19812648857, "ranks": [{"compute_seconds": 0.3172378579999986, "nonpadding_tokens": 5382, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31875791499987827, "nonpadding_tokens": 3693, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31497496900010447, "nonpadding_tokens": 2567, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3204965900001753, "nonpadding_tokens": 9134, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3138790919999792, "nonpadding_tokens": 10314, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3185413730000164, "nonpadding_tokens": 4079, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31423749100008536, "nonpadding_tokens": 2029, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.321358532999966, "nonpadding_tokens": 5308, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003410100000564853, "prefetch": {"data_wait_seconds": 0.02388169600062895, "consumed_batches": 2547, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3097, "tokens": 224541781, "loss_tokens": 58265, "loss": 2.2191710289195914, "lr": 0.00029887770702510694, "optimizer_lrs": [0.019925180468340464, 0.00029887770702510694], "gradient_norm": 0.40989017486572266, "seconds": 0.2543573450000167, "nonpadding_tokens_per_second": 229067.4955739775, "ranks": [{"compute_seconds": 0.2218491799999356, "nonpadding_tokens": 12622, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2227395960001104, "nonpadding_tokens": 7826, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22382485399998586, "nonpadding_tokens": 4966, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22039562300005855, "nonpadding_tokens": 3289, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21899193400008699, "nonpadding_tokens": 7075, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22168790800014904, "nonpadding_tokens": 11478, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2186625310000636, "nonpadding_tokens": 7456, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21940403200005676, "nonpadding_tokens": 3553, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.017002757999989626, "prefetch": {"data_wait_seconds": 0.023889591000809673, "consumed_batches": 2548, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3098, "tokens": 224616605, "loss_tokens": 74824, "loss": 2.4275044604672296, "lr": 0.00029887688691727774, "optimizer_lrs": [0.019925125794485184, 0.00029887688691727774], "gradient_norm": 0.40150153636932373, "seconds": 0.3256007259999478, "nonpadding_tokens_per_second": 229802.92740505742, "ranks": [{"compute_seconds": 0.26184829599992554, "nonpadding_tokens": 12495, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3057119939999211, "nonpadding_tokens": 13627, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3079278519999207, "nonpadding_tokens": 9970, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3081840730001204, "nonpadding_tokens": 4913, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3015843840000798, "nonpadding_tokens": 2933, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30454335600006743, "nonpadding_tokens": 12084, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30199375600000167, "nonpadding_tokens": 8588, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3091091009998763, "nonpadding_tokens": 10214, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0145927619998929, "prefetch": {"data_wait_seconds": 0.023898537000789588, "consumed_batches": 2549, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3099, "tokens": 224682755, "loss_tokens": 66150, "loss": 2.485958994708995, "lr": 0.00029887616163233377, "optimizer_lrs": [0.019925077442155586, 0.00029887616163233377], "gradient_norm": 0.4029095470905304, "seconds": 0.30673281999997926, "nonpadding_tokens_per_second": 215660.00012650902, "ranks": [{"compute_seconds": 0.30176506700013306, "nonpadding_tokens": 2658, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3025937879999674, "nonpadding_tokens": 13507, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.303097464000075, "nonpadding_tokens": 8485, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30020449800008464, "nonpadding_tokens": 3983, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3043445440000596, "nonpadding_tokens": 6456, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3019161930001246, "nonpadding_tokens": 19045, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2987893179999901, "nonpadding_tokens": 9879, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29947202199991807, "nonpadding_tokens": 2137, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004009270001006371, "prefetch": {"data_wait_seconds": 0.023907844000817136, "consumed_batches": 2550, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3100, "tokens": 224754310, "loss_tokens": 71555, "loss": 2.418521591782545, "lr": 0.0002988753768231498, "optimizer_lrs": [0.019925025121543323, 0.0002988753768231498], "gradient_norm": 0.49388548731803894, "seconds": 0.2712376530000711, "nonpadding_tokens_per_second": 263809.24332795804, "ranks": [{"compute_seconds": 0.2683521829999336, "nonpadding_tokens": 10891, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26898629199990864, "nonpadding_tokens": 7375, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2654134919998796, "nonpadding_tokens": 1976, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2664945850001459, "nonpadding_tokens": 5050, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2648137899998346, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2655442069999481, "nonpadding_tokens": 6069, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26854140300019935, "nonpadding_tokens": 3531, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26490374800005156, "nonpadding_tokens": 3895, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003914639999038627, "prefetch": {"data_wait_seconds": 0.023917059000908125, "consumed_batches": 2551, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3101, "tokens": 224795554, "loss_tokens": 41244, "loss": 2.4586803535059647, "lr": 0.00029887492433851797, "optimizer_lrs": [0.0199249949559012, 0.00029887492433851797], "gradient_norm": 0.5115842223167419, "seconds": 0.3076934480000091, "nonpadding_tokens_per_second": 134042.50323847905, "ranks": [{"compute_seconds": 0.30347789100005684, "nonpadding_tokens": 11721, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3047656610001468, "nonpadding_tokens": 1791, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3040664860000106, "nonpadding_tokens": 2166, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3046529120001651, "nonpadding_tokens": 4876, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3053228300000228, "nonpadding_tokens": 1780, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30223028599993995, "nonpadding_tokens": 2203, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30087737999997444, "nonpadding_tokens": 12227, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30173363799985964, "nonpadding_tokens": 4480, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003623289999268309, "prefetch": {"data_wait_seconds": 0.023926205000819323, "consumed_batches": 2552, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3102, "tokens": 224850078, "loss_tokens": 54524, "loss": 2.5587742897623067, "lr": 0.00029887432602096904, "optimizer_lrs": [0.019924955068064604, 0.00029887432602096904], "gradient_norm": 0.4723534882068634, "seconds": 0.2815275510001811, "nonpadding_tokens_per_second": 193671.98629865155, "ranks": [{"compute_seconds": 0.24941291899995122, "nonpadding_tokens": 6962, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2586275979999755, "nonpadding_tokens": 13374, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2590454060000411, "nonpadding_tokens": 2052, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2563885880001635, "nonpadding_tokens": 5483, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2546832269999868, "nonpadding_tokens": 8793, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2572544919999018, "nonpadding_tokens": 8834, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2545085389999713, "nonpadding_tokens": 1744, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2556277400001363, "nonpadding_tokens": 7282, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.006475411000110398, "prefetch": {"data_wait_seconds": 0.023934521000867903, "consumed_batches": 2553, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3103, "tokens": 224911445, "loss_tokens": 61367, "loss": 2.2067858641452247, "lr": 0.0002988736524225067, "optimizer_lrs": [0.01992491016150045, 0.0002988736524225067], "gradient_norm": 0.3912246823310852, "seconds": 0.2809644040000876, "nonpadding_tokens_per_second": 218415.56840054682, "ranks": [{"compute_seconds": 0.27842912299979616, "nonpadding_tokens": 5550, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27702665300012086, "nonpadding_tokens": 1256, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27771690800000215, "nonpadding_tokens": 10144, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2785888809999051, "nonpadding_tokens": 6491, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2729118400000061, "nonpadding_tokens": 11790, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27576869399990755, "nonpadding_tokens": 18653, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27252828899986525, "nonpadding_tokens": 4059, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2734714860000622, "nonpadding_tokens": 3424, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003944580000734277, "prefetch": {"data_wait_seconds": 0.02394462100069461, "consumed_batches": 2554, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3104, "tokens": 224959327, "loss_tokens": 47882, "loss": 2.2582133878597386, "lr": 0.00029887312670358855, "optimizer_lrs": [0.01992487511357257, 0.00029887312670358855], "gradient_norm": 0.3613969385623932, "seconds": 0.2843581880001693, "nonpadding_tokens_per_second": 168386.21858137418, "ranks": [{"compute_seconds": 0.25610659799986024, "nonpadding_tokens": 5163, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2727689419998569, "nonpadding_tokens": 3413, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27573532600013095, "nonpadding_tokens": 10002, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.271356674000117, "nonpadding_tokens": 2467, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2703190120000727, "nonpadding_tokens": 10477, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27377775099989776, "nonpadding_tokens": 9647, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2762273799999093, "nonpadding_tokens": 3779, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2710553230001551, "nonpadding_tokens": 2934, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004956599998422462, "prefetch": {"data_wait_seconds": 0.023954923000658823, "consumed_batches": 2555, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3105, "tokens": 225034206, "loss_tokens": 74879, "loss": 2.3291988157560866, "lr": 0.00029887230432709135, "optimizer_lrs": [0.01992482028847276, 0.00029887230432709135], "gradient_norm": 0.6509053707122803, "seconds": 0.27720212300005187, "nonpadding_tokens_per_second": 270124.1938179023, "ranks": [{"compute_seconds": 0.27417094899988115, "nonpadding_tokens": 4562, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2709759019999183, "nonpadding_tokens": 14096, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26926781099996333, "nonpadding_tokens": 19332, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2700867459998335, "nonpadding_tokens": 19752, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26868720400011625, "nonpadding_tokens": 2614, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27201607500001046, "nonpadding_tokens": 5030, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2747092069998871, "nonpadding_tokens": 5370, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26915974700000334, "nonpadding_tokens": 4123, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037280299989106425, "prefetch": {"data_wait_seconds": 0.023963506000612256, "consumed_batches": 2556, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3106, "tokens": 225092545, "loss_tokens": 58339, "loss": 2.2408688334561786, "lr": 0.00029887166339806807, "optimizer_lrs": [0.019924777559871206, 0.00029887166339806807], "gradient_norm": 0.5244040489196777, "seconds": 0.29299590999994507, "nonpadding_tokens_per_second": 199111.99443026676, "ranks": [{"compute_seconds": 0.286107346000108, "nonpadding_tokens": 5841, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28827692200002275, "nonpadding_tokens": 4825, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28538650599989523, "nonpadding_tokens": 6839, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28580551499999274, "nonpadding_tokens": 13319, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.289996174999942, "nonpadding_tokens": 9873, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28713905800009343, "nonpadding_tokens": 2904, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28365500399991106, "nonpadding_tokens": 10586, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2906011049999506, "nonpadding_tokens": 4152, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039325800003098266, "prefetch": {"data_wait_seconds": 0.023972941000650394, "consumed_batches": 2557, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3107, "tokens": 225166712, "loss_tokens": 74167, "loss": 2.152265748243828, "lr": 0.00029887084831630144, "optimizer_lrs": [0.019924723221086765, 0.00029887084831630144], "gradient_norm": 0.4399547576904297, "seconds": 0.31042716399997516, "nonpadding_tokens_per_second": 238919.1688134803, "ranks": [{"compute_seconds": 0.3077344529999664, "nonpadding_tokens": 5019, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30723434900005486, "nonpadding_tokens": 6954, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3064985789999355, "nonpadding_tokens": 10084, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3070618790000026, "nonpadding_tokens": 12853, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3031095510000341, "nonpadding_tokens": 22423, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.306483771999865, "nonpadding_tokens": 9925, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3033500520000416, "nonpadding_tokens": 4144, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30799814800002423, "nonpadding_tokens": 2765, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037272400004439987, "prefetch": {"data_wait_seconds": 0.023982574000640966, "consumed_batches": 2558, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3108, "tokens": 225232565, "loss_tokens": 65853, "loss": 2.2043505041531897, "lr": 0.00029887012435825494, "optimizer_lrs": [0.019924674957216996, 0.00029887012435825494], "gradient_norm": 0.43903976678848267, "seconds": 0.3371927420000702, "nonpadding_tokens_per_second": 195297.79795789995, "ranks": [{"compute_seconds": 0.2790368830001171, "nonpadding_tokens": 10548, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2972009430000071, "nonpadding_tokens": 2745, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29539823400000387, "nonpadding_tokens": 6126, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29593429800002014, "nonpadding_tokens": 9175, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2940403420000166, "nonpadding_tokens": 11987, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2983275480000884, "nonpadding_tokens": 4489, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29370608100020945, "nonpadding_tokens": 11789, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30264756200017473, "nonpadding_tokens": 8994, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.020094733999940217, "prefetch": {"data_wait_seconds": 0.023992391000732027, "consumed_batches": 2559, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3109, "tokens": 225288429, "loss_tokens": 55864, "loss": 1.986745992499642, "lr": 0.0002988695100336239, "optimizer_lrs": [0.019924634002241596, 0.0002988695100336239], "gradient_norm": 0.40053924918174744, "seconds": 0.262318999999934, "nonpadding_tokens_per_second": 212962.0805203361, "ranks": [{"compute_seconds": 0.25415365699996073, "nonpadding_tokens": 4621, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2557539059998817, "nonpadding_tokens": 1284, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2521332320000056, "nonpadding_tokens": 9559, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2586813309999343, "nonpadding_tokens": 2528, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25131669299980786, "nonpadding_tokens": 2996, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2541646919999039, "nonpadding_tokens": 16437, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.258861195999998, "nonpadding_tokens": 11589, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2595465140000215, "nonpadding_tokens": 6850, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047109600018302444, "prefetch": {"data_wait_seconds": 0.02400283400056935, "consumed_batches": 2560, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3110, "tokens": 225349323, "loss_tokens": 60894, "loss": 2.1997554151476337, "lr": 0.0002988688402057547, "optimizer_lrs": [0.01992458934705031, 0.0002988688402057547], "gradient_norm": 0.4083159863948822, "seconds": 0.2529765840001801, "nonpadding_tokens_per_second": 240710.02555697665, "ranks": [{"compute_seconds": 0.20581731499987654, "nonpadding_tokens": 10470, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2128565789998902, "nonpadding_tokens": 7671, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2099899090001145, "nonpadding_tokens": 2477, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21058267500006878, "nonpadding_tokens": 5593, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.20828923099998065, "nonpadding_tokens": 3068, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21157631799997034, "nonpadding_tokens": 5647, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21380642199983413, "nonpadding_tokens": 11164, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20941888900006234, "nonpadding_tokens": 14804, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.026391646999854856, "prefetch": {"data_wait_seconds": 0.02401363600070283, "consumed_batches": 2561, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3111, "tokens": 225422161, "loss_tokens": 72838, "loss": 2.3978415713638483, "lr": 0.00029886803873561995, "optimizer_lrs": [0.019924535915708, 0.00029886803873561995], "gradient_norm": 0.3925868272781372, "seconds": 0.3013583459999154, "nonpadding_tokens_per_second": 241698.96393053752, "ranks": [{"compute_seconds": 0.26214761799997177, "nonpadding_tokens": 8578, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2643989840000813, "nonpadding_tokens": 7668, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2628669270000046, "nonpadding_tokens": 14599, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26287633300012203, "nonpadding_tokens": 11660, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26210589199990864, "nonpadding_tokens": 14829, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2648973069999556, "nonpadding_tokens": 11209, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.267754750999984, "nonpadding_tokens": 2186, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26222302700011824, "nonpadding_tokens": 2109, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01593153799990432, "prefetch": {"data_wait_seconds": 0.02402394800056129, "consumed_batches": 2562, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3112, "tokens": 225510145, "loss_tokens": 87984, "loss": 2.0442076116112022, "lr": 0.00029886707023014675, "optimizer_lrs": [0.01992447134867645, 0.00029886707023014675], "gradient_norm": 0.39115551114082336, "seconds": 0.3447078890001194, "nonpadding_tokens_per_second": 255242.20015739044, "ranks": [{"compute_seconds": 0.34034668399999646, "nonpadding_tokens": 10948, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3404669790002117, "nonpadding_tokens": 8254, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33729162300005555, "nonpadding_tokens": 18605, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3380149459999302, "nonpadding_tokens": 8626, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.34167908799986435, "nonpadding_tokens": 7205, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3385424850000618, "nonpadding_tokens": 13615, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.341473742000062, "nonpadding_tokens": 6316, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34237308999991, "nonpadding_tokens": 14415, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003698250000070402, "prefetch": {"data_wait_seconds": 0.024033011000668125, "consumed_batches": 2563, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3113, "tokens": 225550933, "loss_tokens": 40788, "loss": 2.229835521476905, "lr": 0.00029886662110631954, "optimizer_lrs": [0.019924441407087972, 0.00029886662110631954], "gradient_norm": 0.4409515857696533, "seconds": 0.25369586500005425, "nonpadding_tokens_per_second": 160775.18646191288, "ranks": [{"compute_seconds": 0.23252530100012336, "nonpadding_tokens": 7348, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22957581000014216, "nonpadding_tokens": 6261, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2287256229999457, "nonpadding_tokens": 3849, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23300697400009085, "nonpadding_tokens": 3965, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22737857599986455, "nonpadding_tokens": 3491, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23032279300014125, "nonpadding_tokens": 5804, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22801902000014707, "nonpadding_tokens": 5969, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2335069260000182, "nonpadding_tokens": 4101, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000455701999953817, "prefetch": {"data_wait_seconds": 0.024042794000664003, "consumed_batches": 2564, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3114, "tokens": 225609243, "loss_tokens": 58310, "loss": 2.193840973889556, "lr": 0.0002988659788907562, "optimizer_lrs": [0.019924398592717083, 0.0002988659788907562], "gradient_norm": 0.4684625267982483, "seconds": 0.28120284900001025, "nonpadding_tokens_per_second": 207359.2078009063, "ranks": [{"compute_seconds": 0.2786940969999705, "nonpadding_tokens": 7073, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2787694510000165, "nonpadding_tokens": 12464, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2787084200001573, "nonpadding_tokens": 3394, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27680035499997757, "nonpadding_tokens": 6867, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27489007400004084, "nonpadding_tokens": 5351, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27786247299991373, "nonpadding_tokens": 12646, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2751018710000608, "nonpadding_tokens": 6425, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27594986700000845, "nonpadding_tokens": 4090, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044207499990989163, "prefetch": {"data_wait_seconds": 0.02405247000065458, "consumed_batches": 2565, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3115, "tokens": 225669052, "loss_tokens": 59809, "loss": 2.460879372251668, "lr": 0.00029886531997730647, "optimizer_lrs": [0.019924354665153767, 0.00029886531997730647], "gradient_norm": 0.5176665186882019, "seconds": 0.2692723790000855, "nonpadding_tokens_per_second": 222113.38653483285, "ranks": [{"compute_seconds": 0.2654541090000748, "nonpadding_tokens": 1159, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26406510099991465, "nonpadding_tokens": 2846, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2615229839998392, "nonpadding_tokens": 11481, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26182749700001295, "nonpadding_tokens": 7304, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2598106039999948, "nonpadding_tokens": 5466, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2634465880000789, "nonpadding_tokens": 1804, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26555319900012364, "nonpadding_tokens": 11156, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26682731500000045, "nonpadding_tokens": 18593, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035487200011630193, "prefetch": {"data_wait_seconds": 0.024061753000708563, "consumed_batches": 2566, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3116, "tokens": 225730546, "loss_tokens": 61494, "loss": 2.3922587976062704, "lr": 0.00029886464230158326, "optimizer_lrs": [0.019924309486772217, 0.00029886464230158326], "gradient_norm": 0.49686044454574585, "seconds": 0.3433059689998572, "nonpadding_tokens_per_second": 179123.01431620485, "ranks": [{"compute_seconds": 0.3386215989999073, "nonpadding_tokens": 10421, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33647802099994806, "nonpadding_tokens": 9379, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3341703129999587, "nonpadding_tokens": 17420, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3407414920000065, "nonpadding_tokens": 6664, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3326846290001413, "nonpadding_tokens": 3574, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.335908462000134, "nonpadding_tokens": 1731, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33283145399991554, "nonpadding_tokens": 9731, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3401453690000835, "nonpadding_tokens": 2574, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003913650000413327, "prefetch": {"data_wait_seconds": 0.024071491000725587, "consumed_batches": 2567, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3117, "tokens": 225789267, "loss_tokens": 58721, "loss": 2.20280462909351, "lr": 0.0002988639949968233, "optimizer_lrs": [0.019924266333121554, 0.0002988639949968233], "gradient_norm": 0.40572962164878845, "seconds": 0.31510780700000396, "nonpadding_tokens_per_second": 186352.0950466304, "ranks": [{"compute_seconds": 0.3107762350000485, "nonpadding_tokens": 9759, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30923802199981765, "nonpadding_tokens": 6497, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3084236160000273, "nonpadding_tokens": 10751, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3098616459999448, "nonpadding_tokens": 5747, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31273601599991707, "nonpadding_tokens": 9003, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3108674479999536, "nonpadding_tokens": 3523, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3123461650000081, "nonpadding_tokens": 9878, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3081617309999274, "nonpadding_tokens": 3563, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036620400010178855, "prefetch": {"data_wait_seconds": 0.02408111000063684, "consumed_batches": 2568, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3118, "tokens": 225869360, "loss_tokens": 80093, "loss": 2.58686113330753, "lr": 0.0002988631118039282, "optimizer_lrs": [0.019924207453595216, 0.0002988631118039282], "gradient_norm": 0.4129638373851776, "seconds": 0.27545036199990136, "nonpadding_tokens_per_second": 290771.0827405944, "ranks": [{"compute_seconds": 0.27096240800005944, "nonpadding_tokens": 2524, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27122706800014384, "nonpadding_tokens": 11451, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26864826400014863, "nonpadding_tokens": 13521, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26961886799995227, "nonpadding_tokens": 4757, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26807952000012847, "nonpadding_tokens": 15367, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27018239399990307, "nonpadding_tokens": 19888, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2728910640000777, "nonpadding_tokens": 2688, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2680991919999087, "nonpadding_tokens": 9897, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003796890000558051, "prefetch": {"data_wait_seconds": 0.024089468000696, "consumed_batches": 2569, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3119, "tokens": 225917941, "loss_tokens": 48581, "loss": 2.656476425969, "lr": 0.00029886257593025645, "optimizer_lrs": [0.019924171728683764, 0.00029886257593025645], "gradient_norm": 0.47713255882263184, "seconds": 0.35260898199999247, "nonpadding_tokens_per_second": 137775.84372482332, "ranks": [{"compute_seconds": 0.3493256649999239, "nonpadding_tokens": 11241, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34751123900014136, "nonpadding_tokens": 5439, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3446777439999096, "nonpadding_tokens": 9655, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34946867499979817, "nonpadding_tokens": 2447, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3440024250000988, "nonpadding_tokens": 4364, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3469153760001973, "nonpadding_tokens": 10126, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.349296835000132, "nonpadding_tokens": 1541, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35032512199995836, "nonpadding_tokens": 3768, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035370299997339316, "prefetch": {"data_wait_seconds": 0.024097684000665254, "consumed_batches": 2570, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3120, "tokens": 225991408, "loss_tokens": 73467, "loss": 2.413679143356881, "lr": 0.00029886176531224216, "optimizer_lrs": [0.019924117687482812, 0.00029886176531224216], "gradient_norm": 0.4907536804676056, "seconds": 0.370359776999976, "nonpadding_tokens_per_second": 198366.5737005905, "ranks": [{"compute_seconds": 0.36771276600006786, "nonpadding_tokens": 20231, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3647272430000612, "nonpadding_tokens": 12145, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36731258500003605, "nonpadding_tokens": 3037, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3645546000000195, "nonpadding_tokens": 3938, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3680446769999435, "nonpadding_tokens": 15639, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36625259899983575, "nonpadding_tokens": 5641, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3629727099998945, "nonpadding_tokens": 2763, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3640192870000192, "nonpadding_tokens": 10073, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037872600000810053, "prefetch": {"data_wait_seconds": 0.024107203000724553, "consumed_batches": 2571, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3121, "tokens": 226058066, "loss_tokens": 66658, "loss": 1.9188572639443127, "lr": 0.0002988610295743794, "optimizer_lrs": [0.01992406863829196, 0.0002988610295743794], "gradient_norm": 0.43751487135887146, "seconds": 0.2667482570000175, "nonpadding_tokens_per_second": 249891.04239955964, "ranks": [{"compute_seconds": 0.2616805630000272, "nonpadding_tokens": 5684, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26372979799998575, "nonpadding_tokens": 18238, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26320466599986503, "nonpadding_tokens": 3047, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2638985759999741, "nonpadding_tokens": 10375, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2594125709999844, "nonpadding_tokens": 8636, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26257728800010227, "nonpadding_tokens": 5437, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2599185250001028, "nonpadding_tokens": 2337, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2643990970000232, "nonpadding_tokens": 12904, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000382958000045619, "prefetch": {"data_wait_seconds": 0.024116809000545913, "consumed_batches": 2572, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3122, "tokens": 226115276, "loss_tokens": 57210, "loss": 2.461469257996854, "lr": 0.00029886039793008913, "optimizer_lrs": [0.01992402652867261, 0.00029886039793008913], "gradient_norm": 0.4216088056564331, "seconds": 0.3570119529999829, "nonpadding_tokens_per_second": 160246.73549236247, "ranks": [{"compute_seconds": 0.35064467899997, "nonpadding_tokens": 8048, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3514341530001275, "nonpadding_tokens": 8999, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3535506670000359, "nonpadding_tokens": 4751, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34985224799993375, "nonpadding_tokens": 3205, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.35458885699995335, "nonpadding_tokens": 10504, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35179937099997005, "nonpadding_tokens": 7994, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.354237219000197, "nonpadding_tokens": 10870, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3486659950001467, "nonpadding_tokens": 2839, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037180599997554964, "prefetch": {"data_wait_seconds": 0.024125409000546938, "consumed_batches": 2573, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3123, "tokens": 226169319, "loss_tokens": 54043, "loss": 2.538735069759266, "lr": 0.0002988598010918739, "optimizer_lrs": [0.01992398673945826, 0.0002988598010918739], "gradient_norm": 0.44839274883270264, "seconds": 0.3043594010000561, "nonpadding_tokens_per_second": 177563.10408821588, "ranks": [{"compute_seconds": 0.3020750800001224, "nonpadding_tokens": 1938, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2972456610000336, "nonpadding_tokens": 11850, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29814429199996084, "nonpadding_tokens": 6138, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29897049700002754, "nonpadding_tokens": 11550, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29728735000003326, "nonpadding_tokens": 5196, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2982010310001897, "nonpadding_tokens": 3857, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30051220900008957, "nonpadding_tokens": 9351, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2977240739999161, "nonpadding_tokens": 4163, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035526300007404643, "prefetch": {"data_wait_seconds": 0.024133552000606073, "consumed_batches": 2574, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3124, "tokens": 226282766, "loss_tokens": 113447, "loss": 2.2422739252690684, "lr": 0.00029885854770370974, "optimizer_lrs": [0.019923903180247317, 0.00029885854770370974], "gradient_norm": 0.46267709136009216, "seconds": 0.2780568250000215, "nonpadding_tokens_per_second": 407999.3361068955, "ranks": [{"compute_seconds": 0.27434720199994445, "nonpadding_tokens": 9736, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2735048800000186, "nonpadding_tokens": 25937, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26983849199996257, "nonpadding_tokens": 14682, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27034790900006556, "nonpadding_tokens": 19286, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27553997399991204, "nonpadding_tokens": 17841, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.272707384999876, "nonpadding_tokens": 9170, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27470672899994497, "nonpadding_tokens": 1971, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2756570029998784, "nonpadding_tokens": 14824, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003595919999952457, "prefetch": {"data_wait_seconds": 0.024142472000633575, "consumed_batches": 2575, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3125, "tokens": 226339872, "loss_tokens": 57106, "loss": 2.405730954278009, "lr": 0.0002988579165242637, "optimizer_lrs": [0.01992386110161758, 0.0002988579165242637], "gradient_norm": 0.4776720106601715, "seconds": 0.22306723899987446, "nonpadding_tokens_per_second": 256003.52725947415, "ranks": [{"compute_seconds": 0.22079509100012729, "nonpadding_tokens": 5328, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22035733799998525, "nonpadding_tokens": 6320, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2170274690001861, "nonpadding_tokens": 4786, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2197633370001313, "nonpadding_tokens": 4690, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22031577799998558, "nonpadding_tokens": 3026, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21793111600004522, "nonpadding_tokens": 13991, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21646165900006054, "nonpadding_tokens": 10927, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21653272200001084, "nonpadding_tokens": 8038, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003909659999408177, "prefetch": {"data_wait_seconds": 0.024152405000634758, "consumed_batches": 2576, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3126, "tokens": 226446505, "loss_tokens": 106633, "loss": 2.5407926603396698, "lr": 0.00029885673746926194, "optimizer_lrs": [0.019923782497950796, 0.00029885673746926194], "gradient_norm": 0.46338823437690735, "seconds": 0.34163754699989113, "nonpadding_tokens_per_second": 312123.18709229573, "ranks": [{"compute_seconds": 0.3391561499997806, "nonpadding_tokens": 14904, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33566015299993524, "nonpadding_tokens": 21305, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33880530199985515, "nonpadding_tokens": 13461, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3358406049999303, "nonpadding_tokens": 10475, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3342540399999052, "nonpadding_tokens": 21437, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33687744400003794, "nonpadding_tokens": 14466, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33931185300002653, "nonpadding_tokens": 5288, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3353124669999943, "nonpadding_tokens": 5297, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039945299999999406, "prefetch": {"data_wait_seconds": 0.024161962000519566, "consumed_batches": 2577, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3127, "tokens": 226495550, "loss_tokens": 49045, "loss": 2.00296968982567, "lr": 0.00029885619496893023, "optimizer_lrs": [0.019923746331262017, 0.00029885619496893023], "gradient_norm": 0.4443013370037079, "seconds": 0.3340994719999344, "nonpadding_tokens_per_second": 146797.59805190482, "ranks": [{"compute_seconds": 0.331000043999893, "nonpadding_tokens": 9835, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32777295000005324, "nonpadding_tokens": 4370, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3284741480001685, "nonpadding_tokens": 5046, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3308719609999571, "nonpadding_tokens": 2240, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32711944199991194, "nonpadding_tokens": 4223, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33017028599988407, "nonpadding_tokens": 10384, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33090687300000354, "nonpadding_tokens": 4581, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3317601730000206, "nonpadding_tokens": 8366, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003904220000094938, "prefetch": {"data_wait_seconds": 0.024170507000420116, "consumed_batches": 2578, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3128, "tokens": 226561697, "loss_tokens": 66147, "loss": 2.223456373682858, "lr": 0.0002988554630956922, "optimizer_lrs": [0.019923697539712817, 0.0002988554630956922], "gradient_norm": 0.3967869281768799, "seconds": 0.3064753749999909, "nonpadding_tokens_per_second": 215831.36981234452, "ranks": [{"compute_seconds": 0.29909880200011685, "nonpadding_tokens": 6926, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30123370699993757, "nonpadding_tokens": 7000, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29813625299993873, "nonpadding_tokens": 4098, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30301724800006014, "nonpadding_tokens": 6267, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29739376199995604, "nonpadding_tokens": 22384, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2998985350000112, "nonpadding_tokens": 2153, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.302988083000173, "nonpadding_tokens": 8884, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30416119599999547, "nonpadding_tokens": 8435, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036814700001741585, "prefetch": {"data_wait_seconds": 0.024180248000405413, "consumed_batches": 2579, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3129, "tokens": 226613392, "loss_tokens": 51695, "loss": 2.279235026356514, "lr": 0.00029885489096211597, "optimizer_lrs": [0.0199236593974744, 0.00029885489096211597], "gradient_norm": 0.40384820103645325, "seconds": 0.2914035750000039, "nonpadding_tokens_per_second": 177400.01988650725, "ranks": [{"compute_seconds": 0.2887501019999945, "nonpadding_tokens": 5311, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28886239100006605, "nonpadding_tokens": 5298, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2879467660000046, "nonpadding_tokens": 1736, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28618217799999, "nonpadding_tokens": 13016, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2887407120001626, "nonpadding_tokens": 8030, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2861663109999881, "nonpadding_tokens": 7471, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2886237230000006, "nonpadding_tokens": 6795, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28907871499995963, "nonpadding_tokens": 4038, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003439990000515536, "prefetch": {"data_wait_seconds": 0.0241883330003958, "consumed_batches": 2580, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3130, "tokens": 226669929, "loss_tokens": 56537, "loss": 2.3150833469674725, "lr": 0.00029885426507680326, "optimizer_lrs": [0.019923617671786884, 0.00029885426507680326], "gradient_norm": 0.4828503131866455, "seconds": 0.2536605669999972, "nonpadding_tokens_per_second": 222884.4659170088, "ranks": [{"compute_seconds": 0.21473540200008756, "nonpadding_tokens": 1984, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21649987599994347, "nonpadding_tokens": 7584, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.218369968999923, "nonpadding_tokens": 11137, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21958885699996245, "nonpadding_tokens": 3086, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21597480799982804, "nonpadding_tokens": 8437, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21723011200015208, "nonpadding_tokens": 10327, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21588936400007697, "nonpadding_tokens": 7865, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21663981200003946, "nonpadding_tokens": 6117, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.02292484700001296, "prefetch": {"data_wait_seconds": 0.02419728600052906, "consumed_batches": 2581, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3131, "tokens": 226723685, "loss_tokens": 53756, "loss": 2.406817233192574, "lr": 0.0002988536698202668, "optimizer_lrs": [0.01992357798801779, 0.0002988536698202668], "gradient_norm": 0.38597071170806885, "seconds": 0.31192601999987346, "nonpadding_tokens_per_second": 172335.73524908826, "ranks": [{"compute_seconds": 0.3069296610001402, "nonpadding_tokens": 5564, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3091226460001053, "nonpadding_tokens": 5174, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3076458560001356, "nonpadding_tokens": 15280, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3091159810001045, "nonpadding_tokens": 1176, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3049587000000429, "nonpadding_tokens": 6311, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30724710100002994, "nonpadding_tokens": 13144, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30842596200000116, "nonpadding_tokens": 3408, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30962288000000626, "nonpadding_tokens": 3699, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003376940001089679, "prefetch": {"data_wait_seconds": 0.02420563700066225, "consumed_batches": 2582, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3132, "tokens": 226800282, "loss_tokens": 76597, "loss": 2.2213282259749074, "lr": 0.0002988528213724568, "optimizer_lrs": [0.019923521424830455, 0.0002988528213724568], "gradient_norm": 0.38614702224731445, "seconds": 0.3389704530000017, "nonpadding_tokens_per_second": 225969.5478531859, "ranks": [{"compute_seconds": 0.33481557499999326, "nonpadding_tokens": 6228, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33670943899983286, "nonpadding_tokens": 4041, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3355818659999841, "nonpadding_tokens": 16151, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3337515109999458, "nonpadding_tokens": 19945, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33293324599981133, "nonpadding_tokens": 3735, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3353909269999349, "nonpadding_tokens": 20667, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3361887940000088, "nonpadding_tokens": 2044, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33337231200016504, "nonpadding_tokens": 3786, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038957999981903413, "prefetch": {"data_wait_seconds": 0.024212774000673676, "consumed_batches": 2583, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3133, "tokens": 226858725, "loss_tokens": 58443, "loss": 2.093789167436648, "lr": 0.0002988521738022375, "optimizer_lrs": [0.019923478253482504, 0.0002988521738022375], "gradient_norm": 0.6217328310012817, "seconds": 0.2727070459998231, "nonpadding_tokens_per_second": 214306.89399949685, "ranks": [{"compute_seconds": 0.26830468299999666, "nonpadding_tokens": 14682, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2701366669998606, "nonpadding_tokens": 6771, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26701798099998086, "nonpadding_tokens": 4424, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2694120600001497, "nonpadding_tokens": 1890, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26991025999996054, "nonpadding_tokens": 4118, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26729508099992927, "nonpadding_tokens": 4523, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26572246999990057, "nonpadding_tokens": 17435, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27033082200000536, "nonpadding_tokens": 4600, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041991800003415847, "prefetch": {"data_wait_seconds": 0.024220885000616477, "consumed_batches": 2584, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3134, "tokens": 226920564, "loss_tokens": 61839, "loss": 2.225971625915684, "lr": 0.0002988514884049658, "optimizer_lrs": [0.019923432560331057, 0.0002988514884049658], "gradient_norm": 0.4810389280319214, "seconds": 0.26570913499995186, "nonpadding_tokens_per_second": 232731.93072572083, "ranks": [{"compute_seconds": 0.2614729309998438, "nonpadding_tokens": 1581, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2626317659999131, "nonpadding_tokens": 12271, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26209073499990154, "nonpadding_tokens": 7127, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26041517199996633, "nonpadding_tokens": 2260, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25869382099995164, "nonpadding_tokens": 6656, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2617614789999152, "nonpadding_tokens": 13953, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2624207240000942, "nonpadding_tokens": 10545, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26315840100005516, "nonpadding_tokens": 7446, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005484190000970557, "prefetch": {"data_wait_seconds": 0.024226874000717658, "consumed_batches": 2585, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3135, "tokens": 227013005, "loss_tokens": 92441, "loss": 2.3050004529916377, "lr": 0.0002988504634482472, "optimizer_lrs": [0.01992336422988315, 0.0002988504634482472], "gradient_norm": 0.5616129040718079, "seconds": 0.28823437699998067, "nonpadding_tokens_per_second": 320714.69393120374, "ranks": [{"compute_seconds": 0.2853317200001584, "nonpadding_tokens": 10205, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28607976099988264, "nonpadding_tokens": 10131, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2825570570000764, "nonpadding_tokens": 14564, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2834245900000951, "nonpadding_tokens": 1735, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28203119299996615, "nonpadding_tokens": 9332, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28441619899990656, "nonpadding_tokens": 20993, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28549531600015143, "nonpadding_tokens": 16069, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2820509870000478, "nonpadding_tokens": 9412, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033429300015086483, "prefetch": {"data_wait_seconds": 0.02423485500067102, "consumed_batches": 2586, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3136, "tokens": 227110865, "loss_tokens": 97860, "loss": 2.2083975194154917, "lr": 0.00029884937791136703, "optimizer_lrs": [0.019923291860757805, 0.00029884937791136703], "gradient_norm": 0.5081763863563538, "seconds": 0.3321642600001269, "nonpadding_tokens_per_second": 294613.2735652012, "ranks": [{"compute_seconds": 0.3292728530000204, "nonpadding_tokens": 10116, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3298532510000314, "nonpadding_tokens": 17874, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3262068439998984, "nonpadding_tokens": 10224, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3290872640000089, "nonpadding_tokens": 15803, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3298262360001445, "nonpadding_tokens": 12048, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3280229459999191, "nonpadding_tokens": 11477, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32949793000011596, "nonpadding_tokens": 5279, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3263417200000731, "nonpadding_tokens": 15039, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003974160001689597, "prefetch": {"data_wait_seconds": 0.02424351300074079, "consumed_batches": 2587, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3137, "tokens": 227168117, "loss_tokens": 57252, "loss": 2.2177876973730175, "lr": 0.0002988487425925125, "optimizer_lrs": [0.0199232495061675, 0.0002988487425925125], "gradient_norm": 0.4106360077857971, "seconds": 0.2646859439998934, "nonpadding_tokens_per_second": 216301.62574867613, "ranks": [{"compute_seconds": 0.26234532900002705, "nonpadding_tokens": 8576, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25895962300000974, "nonpadding_tokens": 4957, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2594116939999367, "nonpadding_tokens": 3112, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2620259979998991, "nonpadding_tokens": 4901, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2581475990000399, "nonpadding_tokens": 17147, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2609938980001516, "nonpadding_tokens": 4336, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2619305460000305, "nonpadding_tokens": 10660, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2592512040000656, "nonpadding_tokens": 3563, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003484839999146061, "prefetch": {"data_wait_seconds": 0.024252907000757773, "consumed_batches": 2588, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3138, "tokens": 227217018, "loss_tokens": 48901, "loss": 2.1806289748675898, "lr": 0.0002988481998054905, "optimizer_lrs": [0.019923213320366034, 0.0002988481998054905], "gradient_norm": 0.44176629185676575, "seconds": 0.27761422200001107, "nonpadding_tokens_per_second": 176147.31568038344, "ranks": [{"compute_seconds": 0.2733468339999945, "nonpadding_tokens": 8800, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27521428099998957, "nonpadding_tokens": 6866, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27433911200000693, "nonpadding_tokens": 4746, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2726601109998228, "nonpadding_tokens": 2852, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2749728939998022, "nonpadding_tokens": 2736, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27260454199995365, "nonpadding_tokens": 18549, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2750175019998551, "nonpadding_tokens": 1455, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2753201909999916, "nonpadding_tokens": 2897, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003584979999686766, "prefetch": {"data_wait_seconds": 0.024262596000838244, "consumed_batches": 2589, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3139, "tokens": 227268754, "loss_tokens": 51736, "loss": 2.485817721219267, "lr": 0.00029884762541211736, "optimizer_lrs": [0.019923175027474493, 0.00029884762541211736], "gradient_norm": 0.47007039189338684, "seconds": 0.33384566600011567, "nonpadding_tokens_per_second": 154969.81170928868, "ranks": [{"compute_seconds": 0.3316225780001787, "nonpadding_tokens": 2741, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32773400400014907, "nonpadding_tokens": 8082, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32855121600005077, "nonpadding_tokens": 3646, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32937941399995907, "nonpadding_tokens": 8074, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33124238699997477, "nonpadding_tokens": 11758, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32868513499988694, "nonpadding_tokens": 2122, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3314071199999944, "nonpadding_tokens": 5493, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3281647339999836, "nonpadding_tokens": 9820, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035381400016376574, "prefetch": {"data_wait_seconds": 0.024272137000892258, "consumed_batches": 2590, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3140, "tokens": 227359918, "loss_tokens": 91164, "loss": 2.260347594719407, "lr": 0.0002988466129266587, "optimizer_lrs": [0.019923107528443916, 0.0002988466129266587], "gradient_norm": 0.5434219837188721, "seconds": 0.27927202199998646, "nonpadding_tokens_per_second": 326434.418124435, "ranks": [{"compute_seconds": 0.2763743240000167, "nonpadding_tokens": 5885, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2769519789999322, "nonpadding_tokens": 10992, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27555141900006674, "nonpadding_tokens": 11989, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27382728299994596, "nonpadding_tokens": 22237, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27214289699986693, "nonpadding_tokens": 10878, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2739415319999807, "nonpadding_tokens": 19375, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2719351480000114, "nonpadding_tokens": 5534, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2733704550000766, "nonpadding_tokens": 4274, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003963819999626139, "prefetch": {"data_wait_seconds": 0.024281848000967443, "consumed_batches": 2591, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3141, "tokens": 227420158, "loss_tokens": 60240, "loss": 2.173812562250996, "lr": 0.0002988459436463677, "optimizer_lrs": [0.019923062909757847, 0.0002988459436463677], "gradient_norm": 0.5044565200805664, "seconds": 0.4053565859999253, "nonpadding_tokens_per_second": 148609.89578200932, "ranks": [{"compute_seconds": 0.4013320130000011, "nonpadding_tokens": 4067, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.399186046000068, "nonpadding_tokens": 8637, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.401943878000111, "nonpadding_tokens": 4088, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4021889269999974, "nonpadding_tokens": 7677, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.40304113199999847, "nonpadding_tokens": 10282, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4019300039999507, "nonpadding_tokens": 15572, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3988316540001051, "nonpadding_tokens": 2026, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3993336679998265, "nonpadding_tokens": 7891, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003391339998870535, "prefetch": {"data_wait_seconds": 0.02429051600093146, "consumed_batches": 2592, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3142, "tokens": 227494480, "loss_tokens": 74322, "loss": 2.3068087595193885, "lr": 0.00029884511764545734, "optimizer_lrs": [0.019923007843030492, 0.00029884511764545734], "gradient_norm": 0.4138992726802826, "seconds": 0.2305232490000435, "nonpadding_tokens_per_second": 322405.65896234603, "ranks": [{"compute_seconds": 0.2252907070001129, "nonpadding_tokens": 8502, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22793924999996307, "nonpadding_tokens": 5191, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22472649500014086, "nonpadding_tokens": 8614, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22653662400011854, "nonpadding_tokens": 11992, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22752990100002535, "nonpadding_tokens": 7180, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22462739699994927, "nonpadding_tokens": 10750, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22751640400019824, "nonpadding_tokens": 9691, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22821319000013318, "nonpadding_tokens": 12402, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000338523999971585, "prefetch": {"data_wait_seconds": 0.024298950000869013, "consumed_batches": 2593, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3143, "tokens": 227580825, "loss_tokens": 86345, "loss": 2.472824497654757, "lr": 0.00029884415765368865, "optimizer_lrs": [0.019922943843579243, 0.00029884415765368865], "gradient_norm": 0.46736881136894226, "seconds": 0.26339278799991916, "nonpadding_tokens_per_second": 327818.3911399522, "ranks": [{"compute_seconds": 0.2610579869999583, "nonpadding_tokens": 3713, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2609277700000803, "nonpadding_tokens": 15171, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25814376700009234, "nonpadding_tokens": 13415, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25851346200010994, "nonpadding_tokens": 23891, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26082400100017367, "nonpadding_tokens": 19719, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2594847719999507, "nonpadding_tokens": 7118, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26092808099997455, "nonpadding_tokens": 1937, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26108348999991904, "nonpadding_tokens": 1381, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003849730001093121, "prefetch": {"data_wait_seconds": 0.024309782000912605, "consumed_batches": 2594, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3144, "tokens": 227645729, "loss_tokens": 64904, "loss": 2.2931204163071612, "lr": 0.0002988434357835175, "optimizer_lrs": [0.019922895718901167, 0.0002988434357835175], "gradient_norm": 0.42686834931373596, "seconds": 0.2898320580000018, "nonpadding_tokens_per_second": 223936.5805421, "ranks": [{"compute_seconds": 0.2875406639998346, "nonpadding_tokens": 11481, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2871006479999778, "nonpadding_tokens": 7254, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2864837519998673, "nonpadding_tokens": 10158, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28703215299992735, "nonpadding_tokens": 4570, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28713250500004506, "nonpadding_tokens": 8097, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2862266780000482, "nonpadding_tokens": 6111, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2837034209999274, "nonpadding_tokens": 2412, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2876036689999637, "nonpadding_tokens": 14821, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003445950001150777, "prefetch": {"data_wait_seconds": 0.024319295000807983, "consumed_batches": 2595, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3145, "tokens": 227705168, "loss_tokens": 59439, "loss": 2.4360518241390334, "lr": 0.0002988427744989056, "optimizer_lrs": [0.019922851633260377, 0.0002988427744989056], "gradient_norm": 0.41643261909484863, "seconds": 0.26457518999995955, "nonpadding_tokens_per_second": 224658.25310381176, "ranks": [{"compute_seconds": 0.2615165489999072, "nonpadding_tokens": 8610, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2588888410000436, "nonpadding_tokens": 10375, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2590885370000251, "nonpadding_tokens": 7449, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2618805950000933, "nonpadding_tokens": 6882, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2622776250000243, "nonpadding_tokens": 9140, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2606199869999273, "nonpadding_tokens": 1607, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2617940489999455, "nonpadding_tokens": 3698, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2589379949999966, "nonpadding_tokens": 11678, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003472690000307921, "prefetch": {"data_wait_seconds": 0.024327826000629216, "consumed_batches": 2596, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3146, "tokens": 227756447, "loss_tokens": 51279, "loss": 2.2798151411884007, "lr": 0.0002988422038466228, "optimizer_lrs": [0.019922813589774855, 0.0002988422038466228], "gradient_norm": 0.37562549114227295, "seconds": 0.3251938819998941, "nonpadding_tokens_per_second": 157687.468425426, "ranks": [{"compute_seconds": 0.3217402550001225, "nonpadding_tokens": 14571, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3223863779999192, "nonpadding_tokens": 6862, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32181528599994635, "nonpadding_tokens": 7478, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3222505960000035, "nonpadding_tokens": 2269, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3228311919999669, "nonpadding_tokens": 6183, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32000525800003743, "nonpadding_tokens": 2512, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3188250089999656, "nonpadding_tokens": 4962, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31919310299986137, "nonpadding_tokens": 6442, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037875400016673666, "prefetch": {"data_wait_seconds": 0.024336128000598478, "consumed_batches": 2597, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3147, "tokens": 227811147, "loss_tokens": 54700, "loss": 2.1399290162248628, "lr": 0.0002988415949697592, "optimizer_lrs": [0.01992277299798395, 0.0002988415949697592], "gradient_norm": 0.3914746642112732, "seconds": 0.2748862400001144, "nonpadding_tokens_per_second": 198991.40822755345, "ranks": [{"compute_seconds": 0.27131690800001707, "nonpadding_tokens": 4264, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27212780899981226, "nonpadding_tokens": 2935, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2708711109999058, "nonpadding_tokens": 5144, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26931911399992714, "nonpadding_tokens": 13180, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2684408719999283, "nonpadding_tokens": 6715, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26944314100001066, "nonpadding_tokens": 5018, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27196596499993575, "nonpadding_tokens": 7482, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27254427400021086, "nonpadding_tokens": 9962, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003746400000181893, "prefetch": {"data_wait_seconds": 0.02434588500041457, "consumed_batches": 2598, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3148, "tokens": 227864412, "loss_tokens": 53265, "loss": 2.307667294424106, "lr": 0.00029884100191301987, "optimizer_lrs": [0.01992273346086799, 0.00029884100191301987], "gradient_norm": 0.4344709515571594, "seconds": 0.27333128600002965, "nonpadding_tokens_per_second": 194873.41086886855, "ranks": [{"compute_seconds": 0.2709176710000065, "nonpadding_tokens": 9824, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26788232900003095, "nonpadding_tokens": 10731, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2708977629999936, "nonpadding_tokens": 5800, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2696836360000816, "nonpadding_tokens": 2200, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2677109119999841, "nonpadding_tokens": 2810, "peak_allocated_bytes": 14981176832, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2690307089999351, "nonpadding_tokens": 14202, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2671910090000438, "nonpadding_tokens": 4714, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2684254570001485, "nonpadding_tokens": 2984, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038177399983396754, "prefetch": {"data_wait_seconds": 0.02435471300027814, "consumed_batches": 2599, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3149, "tokens": 227916183, "loss_tokens": 51771, "loss": 2.3200063138629736, "lr": 0.00029884042534577643, "optimizer_lrs": [0.019922695023051763, 0.00029884042534577643], "gradient_norm": 0.5524533987045288, "seconds": 0.2739400520001709, "nonpadding_tokens_per_second": 188986.60353604556, "ranks": [{"compute_seconds": 0.27100902999995924, "nonpadding_tokens": 3775, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27101605600000767, "nonpadding_tokens": 3398, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2680597599999146, "nonpadding_tokens": 2385, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2707504979998703, "nonpadding_tokens": 16112, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2667719250000573, "nonpadding_tokens": 13193, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27037949800001115, "nonpadding_tokens": 4315, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2708333189998484, "nonpadding_tokens": 4963, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2716223480001645, "nonpadding_tokens": 3630, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034550899999885587, "prefetch": {"data_wait_seconds": 0.02436352800027919, "consumed_batches": 2600, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3150, "tokens": 227984195, "loss_tokens": 68012, "loss": 2.3073896885843674, "lr": 0.00029883966768755885, "optimizer_lrs": [0.019922644512503924, 0.00029883966768755885], "gradient_norm": 0.4521774649620056, "seconds": 0.24847576899992418, "nonpadding_tokens_per_second": 273716.83071447, "ranks": [{"compute_seconds": 0.2450655249999727, "nonpadding_tokens": 5177, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24200364899979832, "nonpadding_tokens": 6828, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24294362599994201, "nonpadding_tokens": 20972, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24527441899999758, "nonpadding_tokens": 2972, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2416379680000773, "nonpadding_tokens": 5570, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24424246000012317, "nonpadding_tokens": 11590, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24191761899987796, "nonpadding_tokens": 7575, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24619854499997018, "nonpadding_tokens": 7328, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034999400008928205, "prefetch": {"data_wait_seconds": 0.024372563000227387, "consumed_batches": 2601, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3151, "tokens": 228039647, "loss_tokens": 55452, "loss": 2.1489201809673233, "lr": 0.00029883904976625114, "optimizer_lrs": [0.019922603317750077, 0.00029883904976625114], "gradient_norm": 0.39575836062431335, "seconds": 0.25193275499987067, "nonpadding_tokens_per_second": 220106.35338000598, "ranks": [{"compute_seconds": 0.24770968300003915, "nonpadding_tokens": 6906, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24960952800006453, "nonpadding_tokens": 4274, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24617556099997273, "nonpadding_tokens": 9905, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2492286100000456, "nonpadding_tokens": 7386, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24587466200000563, "nonpadding_tokens": 8264, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24862670500010609, "nonpadding_tokens": 5284, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24583875099983743, "nonpadding_tokens": 5518, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24945992500011016, "nonpadding_tokens": 7915, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000393675999930565, "prefetch": {"data_wait_seconds": 0.024380721000355265, "consumed_batches": 2602, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3152, "tokens": 228097926, "loss_tokens": 58279, "loss": 2.6067499656823214, "lr": 0.00029883840016616323, "optimizer_lrs": [0.01992256001107755, 0.00029883840016616323], "gradient_norm": 0.43355634808540344, "seconds": 0.2482587560000411, "nonpadding_tokens_per_second": 234751.03532698905, "ranks": [{"compute_seconds": 0.24417907800011562, "nonpadding_tokens": 7908, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2418598669999028, "nonpadding_tokens": 6778, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2429660210000293, "nonpadding_tokens": 2658, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24480445999984113, "nonpadding_tokens": 12825, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24171897799988074, "nonpadding_tokens": 15449, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24413225500006774, "nonpadding_tokens": 2342, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24541358300007232, "nonpadding_tokens": 6745, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24569301600013205, "nonpadding_tokens": 3574, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003711589999966236, "prefetch": {"data_wait_seconds": 0.024390409000218938, "consumed_batches": 2603, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3153, "tokens": 228152703, "loss_tokens": 54777, "loss": 2.0892825969384963, "lr": 0.00029883778943578664, "optimizer_lrs": [0.019922519295719112, 0.00029883778943578664], "gradient_norm": 0.4366177022457123, "seconds": 0.2392848409999715, "nonpadding_tokens_per_second": 228919.6414243664, "ranks": [{"compute_seconds": 0.23647405999986404, "nonpadding_tokens": 2073, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23700078799993207, "nonpadding_tokens": 6690, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2329654559998744, "nonpadding_tokens": 3206, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2359820579999905, "nonpadding_tokens": 7290, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23232489799988798, "nonpadding_tokens": 1885, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2335581950001142, "nonpadding_tokens": 18673, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23608747599996605, "nonpadding_tokens": 10736, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23258505500007232, "nonpadding_tokens": 4224, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003617069999108935, "prefetch": {"data_wait_seconds": 0.024398638000278083, "consumed_batches": 2604, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3154, "tokens": 228211369, "loss_tokens": 58666, "loss": 2.490402501022739, "lr": 0.00029883713516817845, "optimizer_lrs": [0.019922475677878564, 0.00029883713516817845], "gradient_norm": 0.41677361726760864, "seconds": 0.27042871200001173, "nonpadding_tokens_per_second": 216937.02405385658, "ranks": [{"compute_seconds": 0.26684035799985395, "nonpadding_tokens": 6391, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2678786999999829, "nonpadding_tokens": 7579, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26491333400008443, "nonpadding_tokens": 3029, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2677106730000105, "nonpadding_tokens": 12160, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2635679329998766, "nonpadding_tokens": 14168, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2651714579999407, "nonpadding_tokens": 5598, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2677660009999272, "nonpadding_tokens": 5074, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2681071399999837, "nonpadding_tokens": 4667, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003578890000426327, "prefetch": {"data_wait_seconds": 0.024407639000358472, "consumed_batches": 2605, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3155, "tokens": 228264005, "loss_tokens": 52636, "loss": 2.116265928974466, "lr": 0.00029883654799361817, "optimizer_lrs": [0.01992243653290788, 0.00029883654799361817], "gradient_norm": 0.3957091271877289, "seconds": 0.27075349599999754, "nonpadding_tokens_per_second": 194405.61535722692, "ranks": [{"compute_seconds": 0.2683440559999326, "nonpadding_tokens": 2693, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26748223399999915, "nonpadding_tokens": 10377, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.266986389000067, "nonpadding_tokens": 9826, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2675292349999836, "nonpadding_tokens": 5691, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2635126160000709, "nonpadding_tokens": 4165, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2664409809999597, "nonpadding_tokens": 2921, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26789366700018036, "nonpadding_tokens": 9584, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26839343899996493, "nonpadding_tokens": 7379, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003474430000096618, "prefetch": {"data_wait_seconds": 0.024416075000274873, "consumed_batches": 2606, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3156, "tokens": 228317203, "loss_tokens": 53198, "loss": 2.2499198160645135, "lr": 0.0002988359543997984, "optimizer_lrs": [0.01992239695998656, 0.0002988359543997984], "gradient_norm": 0.37843215465545654, "seconds": 0.45886753399986446, "nonpadding_tokens_per_second": 115933.24011459855, "ranks": [{"compute_seconds": 0.4564718010001343, "nonpadding_tokens": 12178, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.45254286199997296, "nonpadding_tokens": 5421, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4553130299998429, "nonpadding_tokens": 4294, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4532798230000026, "nonpadding_tokens": 2312, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.45202502500001174, "nonpadding_tokens": 5653, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.45481888900008016, "nonpadding_tokens": 4572, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.45579297399990537, "nonpadding_tokens": 14763, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.4528376099999605, "nonpadding_tokens": 4005, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003721699999914563, "prefetch": {"data_wait_seconds": 0.02442506800021249, "consumed_batches": 2607, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3157, "tokens": 228366634, "loss_tokens": 49431, "loss": 2.0408666246889604, "lr": 0.00029883540270381133, "optimizer_lrs": [0.01992236018025409, 0.00029883540270381133], "gradient_norm": 0.39008745551109314, "seconds": 0.29895816699990974, "nonpadding_tokens_per_second": 165344.20349190503, "ranks": [{"compute_seconds": 0.2943213769999602, "nonpadding_tokens": 3107, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2922784849999971, "nonpadding_tokens": 3744, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2928436810000221, "nonpadding_tokens": 9934, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29569915799993396, "nonpadding_tokens": 5339, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2959885049999684, "nonpadding_tokens": 6623, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29479360699997414, "nonpadding_tokens": 3012, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29551427199999125, "nonpadding_tokens": 12272, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29656515199985733, "nonpadding_tokens": 5400, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004210130000501522, "prefetch": {"data_wait_seconds": 0.02443490200012377, "consumed_batches": 2608, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3158, "tokens": 228440064, "loss_tokens": 73430, "loss": 2.2868492952471744, "lr": 0.00029883458291638453, "optimizer_lrs": [0.01992230552775897, 0.00029883458291638453], "gradient_norm": 0.4862504303455353, "seconds": 0.3379676479999034, "nonpadding_tokens_per_second": 217269.31685491087, "ranks": [{"compute_seconds": 0.33509223500004737, "nonpadding_tokens": 14505, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3319453470001008, "nonpadding_tokens": 11120, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.335026925999955, "nonpadding_tokens": 12434, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3338033280001582, "nonpadding_tokens": 6604, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.332239890999972, "nonpadding_tokens": 6833, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3329300439997951, "nonpadding_tokens": 14364, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33538885100006155, "nonpadding_tokens": 4491, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3324343240001326, "nonpadding_tokens": 3079, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003713900000548165, "prefetch": {"data_wait_seconds": 0.02444417000010901, "consumed_batches": 2609, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3159, "tokens": 228508009, "loss_tokens": 67945, "loss": 2.5749983902421074, "lr": 0.0002988338241088059, "optimizer_lrs": [0.019922254940587063, 0.0002988338241088059], "gradient_norm": 0.44787460565567017, "seconds": 0.25714507800012143, "nonpadding_tokens_per_second": 264228.2735039086, "ranks": [{"compute_seconds": 0.25322987399999874, "nonpadding_tokens": 2409, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2547950170001059, "nonpadding_tokens": 8373, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25315222800008996, "nonpadding_tokens": 2439, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25200367200000073, "nonpadding_tokens": 2736, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2503795869999976, "nonpadding_tokens": 15682, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2513889110000491, "nonpadding_tokens": 15985, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25396257700003844, "nonpadding_tokens": 13320, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.251219945999992, "nonpadding_tokens": 7001, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003790570001456217, "prefetch": {"data_wait_seconds": 0.024453553000242323, "consumed_batches": 2610, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3160, "tokens": 228580629, "loss_tokens": 72620, "loss": 2.252138701459653, "lr": 0.0002988330128191392, "optimizer_lrs": [0.019922200854609284, 0.0002988330128191392], "gradient_norm": 0.39297106862068176, "seconds": 0.25274939699988863, "nonpadding_tokens_per_second": 287320.1711339078, "ranks": [{"compute_seconds": 0.25017225500005225, "nonpadding_tokens": 7240, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24670141000001422, "nonpadding_tokens": 1643, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.247263976999875, "nonpadding_tokens": 18931, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25036600099997486, "nonpadding_tokens": 4217, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24607901600006699, "nonpadding_tokens": 7476, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24781738699994094, "nonpadding_tokens": 2844, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2500891900001534, "nonpadding_tokens": 14417, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24714859899995645, "nonpadding_tokens": 15852, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047277699991354893, "prefetch": {"data_wait_seconds": 0.024463425000206485, "consumed_batches": 2611, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3161, "tokens": 228662242, "loss_tokens": 81613, "loss": 2.0904305992917793, "lr": 0.0002988321007270686, "optimizer_lrs": [0.01992214004847124, 0.0002988321007270686], "gradient_norm": 0.45905202627182007, "seconds": 0.2520039319999796, "nonpadding_tokens_per_second": 323856.05792851915, "ranks": [{"compute_seconds": 0.24855869499992878, "nonpadding_tokens": 11196, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2495408419999876, "nonpadding_tokens": 10683, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24853028400002586, "nonpadding_tokens": 18065, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24714923999999883, "nonpadding_tokens": 12948, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2455578730000525, "nonpadding_tokens": 3236, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24776835300008315, "nonpadding_tokens": 6247, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24885655899993253, "nonpadding_tokens": 12985, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24964271699991514, "nonpadding_tokens": 6253, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004276390000086394, "prefetch": {"data_wait_seconds": 0.024472926000228767, "consumed_batches": 2612, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3162, "tokens": 228760219, "loss_tokens": 97977, "loss": 2.2612472187350092, "lr": 0.00029883100528533174, "optimizer_lrs": [0.01992206701902212, 0.00029883100528533174], "gradient_norm": 0.3571045994758606, "seconds": 0.257469484000012, "nonpadding_tokens_per_second": 380538.3009972375, "ranks": [{"compute_seconds": 0.25528737199988427, "nonpadding_tokens": 10061, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25227832599989597, "nonpadding_tokens": 17351, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25337328900013745, "nonpadding_tokens": 5888, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2550394249999499, "nonpadding_tokens": 13448, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.252298638999946, "nonpadding_tokens": 17772, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2528747910000675, "nonpadding_tokens": 19461, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2522305560000859, "nonpadding_tokens": 9772, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2525534559999869, "nonpadding_tokens": 4224, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033462700002928614, "prefetch": {"data_wait_seconds": 0.024481648000303835, "consumed_batches": 2613, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3163, "tokens": 228808857, "loss_tokens": 48638, "loss": 2.1369081273901065, "lr": 0.0002988304612933649, "optimizer_lrs": [0.019922030752890996, 0.0002988304612933649], "gradient_norm": 0.42275384068489075, "seconds": 0.21897145699995235, "nonpadding_tokens_per_second": 222120.27387665684, "ranks": [{"compute_seconds": 0.21626400999980433, "nonpadding_tokens": 1968, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21282030799989116, "nonpadding_tokens": 1841, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2136120350000965, "nonpadding_tokens": 5513, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2157892620000439, "nonpadding_tokens": 2846, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21666811999989477, "nonpadding_tokens": 5013, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2148827840001104, "nonpadding_tokens": 15167, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2162735500000963, "nonpadding_tokens": 2325, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21327518600014628, "nonpadding_tokens": 13965, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038702200004081533, "prefetch": {"data_wait_seconds": 0.024490235000214966, "consumed_batches": 2614, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3164, "tokens": 228859943, "loss_tokens": 51086, "loss": 2.4023247868789883, "lr": 0.0002988298897860644, "optimizer_lrs": [0.019921992652404293, 0.0002988298897860644], "gradient_norm": 0.41122451424598694, "seconds": 0.25882291699986126, "nonpadding_tokens_per_second": 197378.194296556, "ranks": [{"compute_seconds": 0.25564661099997465, "nonpadding_tokens": 2933, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2528788730000997, "nonpadding_tokens": 2982, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2529539649999606, "nonpadding_tokens": 10942, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25546285100017485, "nonpadding_tokens": 18927, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25180215800014594, "nonpadding_tokens": 4886, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2551083830001062, "nonpadding_tokens": 3849, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25248845999999503, "nonpadding_tokens": 3142, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25652416700017966, "nonpadding_tokens": 3425, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000351478999846222, "prefetch": {"data_wait_seconds": 0.024499060000337636, "consumed_batches": 2615, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3165, "tokens": 228921171, "loss_tokens": 61228, "loss": 2.616334897840857, "lr": 0.0002988292046354531, "optimizer_lrs": [0.019921946975696873, 0.0002988292046354531], "gradient_norm": 0.4296201765537262, "seconds": 0.2984229910000522, "nonpadding_tokens_per_second": 205171.85956355918, "ranks": [{"compute_seconds": 0.2950089979999575, "nonpadding_tokens": 16118, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29442522199997256, "nonpadding_tokens": 2036, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2947054080000271, "nonpadding_tokens": 10780, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29528667800013864, "nonpadding_tokens": 6425, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29611604100000477, "nonpadding_tokens": 9327, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29306213100016976, "nonpadding_tokens": 3968, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29572628899995834, "nonpadding_tokens": 4866, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2959698450001724, "nonpadding_tokens": 7708, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034081300009347615, "prefetch": {"data_wait_seconds": 0.024509401000344155, "consumed_batches": 2616, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3166, "tokens": 228976432, "loss_tokens": 55261, "loss": 2.176157377038056, "lr": 0.0002988285860850556, "optimizer_lrs": [0.01992190573900371, 0.0002988285860850556], "gradient_norm": 0.4544251263141632, "seconds": 0.3090339320001476, "nonpadding_tokens_per_second": 178818.5512261922, "ranks": [{"compute_seconds": 0.30671779099998275, "nonpadding_tokens": 6951, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3066210049998972, "nonpadding_tokens": 2752, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3027428149998741, "nonpadding_tokens": 3663, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30386063900004956, "nonpadding_tokens": 8178, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3064349169999332, "nonpadding_tokens": 12380, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.305068497000093, "nonpadding_tokens": 2755, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30228266000017356, "nonpadding_tokens": 10191, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3030240410000715, "nonpadding_tokens": 8391, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037672500002372544, "prefetch": {"data_wait_seconds": 0.024518784000250093, "consumed_batches": 2617, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3167, "tokens": 229024708, "loss_tokens": 48276, "loss": 2.284966909023117, "lr": 0.00029882804558643307, "optimizer_lrs": [0.019921869705762207, 0.00029882804558643307], "gradient_norm": 0.5276572108268738, "seconds": 0.3492824649999875, "nonpadding_tokens_per_second": 138214.78269744152, "ranks": [{"compute_seconds": 0.3461876439998832, "nonpadding_tokens": 8924, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.343198091999966, "nonpadding_tokens": 2398, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34566828199990596, "nonpadding_tokens": 11952, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34408805099997153, "nonpadding_tokens": 10111, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.34689164300016273, "nonpadding_tokens": 1170, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3457220899999811, "nonpadding_tokens": 3643, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3464540269999361, "nonpadding_tokens": 5541, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.343916469000078, "nonpadding_tokens": 4537, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040671300007488753, "prefetch": {"data_wait_seconds": 0.024528781000256572, "consumed_batches": 2618, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3168, "tokens": 229103227, "loss_tokens": 78519, "loss": 2.235115067690623, "lr": 0.0002988271662217375, "optimizer_lrs": [0.019921811081449168, 0.0002988271662217375], "gradient_norm": 0.40243467688560486, "seconds": 0.25811271199995645, "nonpadding_tokens_per_second": 304204.3121069266, "ranks": [{"compute_seconds": 0.2555461130000367, "nonpadding_tokens": 10597, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25577229499981513, "nonpadding_tokens": 3276, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.254466814999887, "nonpadding_tokens": 20080, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2548490869999114, "nonpadding_tokens": 12046, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2511046049999095, "nonpadding_tokens": 3495, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2523476040000787, "nonpadding_tokens": 9429, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25156029599997964, "nonpadding_tokens": 9764, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2517854029999853, "nonpadding_tokens": 9832, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040543699992667825, "prefetch": {"data_wait_seconds": 0.0245386180003635, "consumed_batches": 2619, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3169, "tokens": 229155312, "loss_tokens": 52085, "loss": 2.4235297470480943, "lr": 0.00029882658272050285, "optimizer_lrs": [0.019921772181366856, 0.00029882658272050285], "gradient_norm": 0.4234533905982971, "seconds": 0.28388431799999125, "nonpadding_tokens_per_second": 183472.6214077158, "ranks": [{"compute_seconds": 0.28141264499981844, "nonpadding_tokens": 10383, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28106834999994135, "nonpadding_tokens": 3178, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2787247670000852, "nonpadding_tokens": 7337, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2790903800000706, "nonpadding_tokens": 3665, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28152058299997407, "nonpadding_tokens": 4338, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2801434979999158, "nonpadding_tokens": 5666, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2769440489998942, "nonpadding_tokens": 4293, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28158344399980706, "nonpadding_tokens": 13225, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035668000009536627, "prefetch": {"data_wait_seconds": 0.024546845000216422, "consumed_batches": 2620, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3170, "tokens": 229231560, "loss_tokens": 76248, "loss": 2.4724068582126746, "lr": 0.00029882572826386866, "optimizer_lrs": [0.019921715217591247, 0.00029882572826386866], "gradient_norm": 0.3956753611564636, "seconds": 0.3374280319999343, "nonpadding_tokens_per_second": 225968.18512107152, "ranks": [{"compute_seconds": 0.33399775100019724, "nonpadding_tokens": 8321, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3313529209999615, "nonpadding_tokens": 18232, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3338647220000439, "nonpadding_tokens": 4076, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3346638959999382, "nonpadding_tokens": 2622, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3310052569997879, "nonpadding_tokens": 11113, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33388001299999814, "nonpadding_tokens": 1816, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33086504700008845, "nonpadding_tokens": 4166, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3349961449998773, "nonpadding_tokens": 25902, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003796240000610851, "prefetch": {"data_wait_seconds": 0.024555505000265043, "consumed_batches": 2621, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3171, "tokens": 229380221, "loss_tokens": 148661, "loss": 4.365641039008214, "lr": 0.00029882406143635855, "optimizer_lrs": [0.019921604095757238, 0.00029882406143635855], "gradient_norm": 1.735034465789795, "seconds": 0.31667781300006936, "nonpadding_tokens_per_second": 469439.2657055751, "ranks": [{"compute_seconds": 0.27741556799992395, "nonpadding_tokens": 14710, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2872102320000067, "nonpadding_tokens": 4931, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2856295080000564, "nonpadding_tokens": 4253, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2862515900001199, "nonpadding_tokens": 31794, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2869930450001448, "nonpadding_tokens": 25065, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.285640494999825, "nonpadding_tokens": 20569, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28325238100001116, "nonpadding_tokens": 25640, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28325431899997966, "nonpadding_tokens": 21699, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.01859203799995157, "prefetch": {"data_wait_seconds": 0.024564572000329576, "consumed_batches": 2622, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3172, "tokens": 229507968, "loss_tokens": 127747, "loss": 4.665711327858971, "lr": 0.0002988226281620394, "optimizer_lrs": [0.01992150854413596, 0.0002988226281620394], "gradient_norm": 1.842329978942871, "seconds": 0.3371870570001647, "nonpadding_tokens_per_second": 378860.92407140526, "ranks": [{"compute_seconds": 0.3330784710001353, "nonpadding_tokens": 25860, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3348773220000112, "nonpadding_tokens": 10370, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33156186099995466, "nonpadding_tokens": 10717, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33248580800000127, "nonpadding_tokens": 18580, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33105747399986285, "nonpadding_tokens": 19866, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33053825599995434, "nonpadding_tokens": 18887, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33280894299991814, "nonpadding_tokens": 13168, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3338998019999053, "nonpadding_tokens": 10299, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036715099986395217, "prefetch": {"data_wait_seconds": 0.0245742030003413, "consumed_batches": 2623, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3173, "tokens": 229572376, "loss_tokens": 64408, "loss": 2.258533732222705, "lr": 0.0002988219051984278, "optimizer_lrs": [0.019921460346561854, 0.0002988219051984278], "gradient_norm": 0.5942270755767822, "seconds": 0.35822829700009606, "nonpadding_tokens_per_second": 179795.95844150393, "ranks": [{"compute_seconds": 0.3555359640001825, "nonpadding_tokens": 7561, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35506221299988283, "nonpadding_tokens": 16014, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35471007800015286, "nonpadding_tokens": 10903, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3555541539999467, "nonpadding_tokens": 5570, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.35583389499993245, "nonpadding_tokens": 3352, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3531953919998614, "nonpadding_tokens": 6907, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35537341600002037, "nonpadding_tokens": 6015, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35121634099982657, "nonpadding_tokens": 8086, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036571500004356494, "prefetch": {"data_wait_seconds": 0.024584260000395375, "consumed_batches": 2624, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3174, "tokens": 229660527, "loss_tokens": 88151, "loss": 2.1500057429864663, "lr": 0.00029882091536750975, "optimizer_lrs": [0.019921394357833983, 0.00029882091536750975], "gradient_norm": 0.6039363145828247, "seconds": 0.3444525109998722, "nonpadding_tokens_per_second": 255916.26475335148, "ranks": [{"compute_seconds": 0.34137702099997114, "nonpadding_tokens": 8442, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33859344000006786, "nonpadding_tokens": 9018, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3392015040001297, "nonpadding_tokens": 17604, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3419164780000301, "nonpadding_tokens": 16660, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3420545349999884, "nonpadding_tokens": 9148, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.339760662000117, "nonpadding_tokens": 12910, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34208527200007666, "nonpadding_tokens": 11252, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3389456310001151, "nonpadding_tokens": 3117, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004051340001751669, "prefetch": {"data_wait_seconds": 0.024593711000264307, "consumed_batches": 2625, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3175, "tokens": 229715686, "loss_tokens": 55159, "loss": 2.366836809269566, "lr": 0.0002988202957870356, "optimizer_lrs": [0.01992135305246904, 0.0002988202957870356], "gradient_norm": 0.43966707587242126, "seconds": 0.29583819100002984, "nonpadding_tokens_per_second": 186449.8961866436, "ranks": [{"compute_seconds": 0.2933028219999869, "nonpadding_tokens": 1888, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.289647142999911, "nonpadding_tokens": 5411, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29212862999997924, "nonpadding_tokens": 4408, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2904331209999782, "nonpadding_tokens": 8096, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29294446099993365, "nonpadding_tokens": 3161, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2911107530001118, "nonpadding_tokens": 11937, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2883997730000374, "nonpadding_tokens": 11433, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2935021920000054, "nonpadding_tokens": 8825, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038688500012540317, "prefetch": {"data_wait_seconds": 0.024603056000344736, "consumed_batches": 2626, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3176, "tokens": 229762628, "loss_tokens": 46942, "loss": 2.4932483170721316, "lr": 0.000298819768377436, "optimizer_lrs": [0.019921317891829066, 0.000298819768377436], "gradient_norm": 0.465498149394989, "seconds": 0.2507354840001881, "nonpadding_tokens_per_second": 187217.21892368767, "ranks": [{"compute_seconds": 0.24636712500000613, "nonpadding_tokens": 5210, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24369025399983002, "nonpadding_tokens": 12267, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2462357689998953, "nonpadding_tokens": 6288, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24405641299995295, "nonpadding_tokens": 2738, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24234879400000864, "nonpadding_tokens": 4468, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24465424100003474, "nonpadding_tokens": 8528, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24727843599998778, "nonpadding_tokens": 2250, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2483800259999498, "nonpadding_tokens": 5193, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042021599983854685, "prefetch": {"data_wait_seconds": 0.024616114000266975, "consumed_batches": 2627, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3177, "tokens": 229846203, "loss_tokens": 83575, "loss": 2.269099985043374, "lr": 0.0002988188290929639, "optimizer_lrs": [0.01992125527286426, 0.0002988188290929639], "gradient_norm": 0.3946335017681122, "seconds": 0.2523851600001308, "nonpadding_tokens_per_second": 331140.7057370437, "ranks": [{"compute_seconds": 0.248812437999959, "nonpadding_tokens": 8969, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2464378449999458, "nonpadding_tokens": 10084, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2489074459999756, "nonpadding_tokens": 5722, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2463805830000183, "nonpadding_tokens": 5813, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2500580309999805, "nonpadding_tokens": 14114, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2482968400001937, "nonpadding_tokens": 10389, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24940087899994978, "nonpadding_tokens": 16730, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2460926969999946, "nonpadding_tokens": 11754, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037578000001303735, "prefetch": {"data_wait_seconds": 0.02462452300024154, "consumed_batches": 2628, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3178, "tokens": 229920054, "loss_tokens": 73851, "loss": 1.8718200329040906, "lr": 0.00029881799878521637, "optimizer_lrs": [0.019921199919014424, 0.00029881799878521637], "gradient_norm": 0.5048028230667114, "seconds": 0.3689023710001038, "nonpadding_tokens_per_second": 200191.17741040277, "ranks": [{"compute_seconds": 0.36659223199990265, "nonpadding_tokens": 18004, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3643429820001529, "nonpadding_tokens": 14656, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3647063169999001, "nonpadding_tokens": 4372, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.36259646799999246, "nonpadding_tokens": 5138, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3608895540000958, "nonpadding_tokens": 3621, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36277365699993425, "nonpadding_tokens": 4845, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36527743700003157, "nonpadding_tokens": 13604, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.36606333800000357, "nonpadding_tokens": 9611, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003427359999932378, "prefetch": {"data_wait_seconds": 0.024633734000147456, "consumed_batches": 2629, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3179, "tokens": 230034500, "loss_tokens": 114446, "loss": 2.4551565257850863, "lr": 0.0002988167114935478, "optimizer_lrs": [0.019921114099569856, 0.0002988167114935478], "gradient_norm": 0.4120544195175171, "seconds": 0.3697820459999548, "nonpadding_tokens_per_second": 309495.8266308419, "ranks": [{"compute_seconds": 0.3665808040000229, "nonpadding_tokens": 20751, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36711663200003386, "nonpadding_tokens": 14530, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3665160329999253, "nonpadding_tokens": 9998, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3671958010002072, "nonpadding_tokens": 18694, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.363497020000068, "nonpadding_tokens": 13420, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36455003800006125, "nonpadding_tokens": 9628, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3666963390001001, "nonpadding_tokens": 25043, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3674489069999254, "nonpadding_tokens": 2382, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037196399989625206, "prefetch": {"data_wait_seconds": 0.02464296500006924, "consumed_batches": 2630, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3180, "tokens": 230098460, "loss_tokens": 63960, "loss": 2.0806961870700436, "lr": 0.00029881599176592087, "optimizer_lrs": [0.01992106611772806, 0.00029881599176592087], "gradient_norm": 0.3971143662929535, "seconds": 0.29399757200008025, "nonpadding_tokens_per_second": 217552.81706878362, "ranks": [{"compute_seconds": 0.2878884060000928, "nonpadding_tokens": 12830, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28798724000012044, "nonpadding_tokens": 14721, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2905415570000969, "nonpadding_tokens": 8243, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2879583849999108, "nonpadding_tokens": 2343, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28602202100000795, "nonpadding_tokens": 4118, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2886903319999874, "nonpadding_tokens": 4744, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2914913560000514, "nonpadding_tokens": 12439, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2917373249999855, "nonpadding_tokens": 4522, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037250900004437426, "prefetch": {"data_wait_seconds": 0.02465185199999098, "consumed_batches": 2631, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3181, "tokens": 230146949, "loss_tokens": 48489, "loss": 2.5523872037987996, "lr": 0.00029881544598482553, "optimizer_lrs": [0.019921029732321703, 0.00029881544598482553], "gradient_norm": 0.4543306827545166, "seconds": 0.34331349600006433, "nonpadding_tokens_per_second": 141238.2576419044, "ranks": [{"compute_seconds": 0.34089368800005104, "nonpadding_tokens": 1386, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34041517700006807, "nonpadding_tokens": 7808, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3374587230000543, "nonpadding_tokens": 11035, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3386203079999177, "nonpadding_tokens": 6998, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33651019200010523, "nonpadding_tokens": 3720, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33948180099991987, "nonpadding_tokens": 7161, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34030774299981204, "nonpadding_tokens": 5256, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34063355399985085, "nonpadding_tokens": 5125, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037567599997601064, "prefetch": {"data_wait_seconds": 0.024660384000029012, "consumed_batches": 2632, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3182, "tokens": 230216554, "loss_tokens": 69605, "loss": 1.8732838427555492, "lr": 0.00029881466230799355, "optimizer_lrs": [0.019920977487199572, 0.00029881466230799355], "gradient_norm": 0.912979781627655, "seconds": 0.257538908000015, "nonpadding_tokens_per_second": 270269.8421008912, "ranks": [{"compute_seconds": 0.2523763249998865, "nonpadding_tokens": 5121, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2552217049999399, "nonpadding_tokens": 12661, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2538509779999458, "nonpadding_tokens": 8693, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25428933000011966, "nonpadding_tokens": 11608, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2549125870000353, "nonpadding_tokens": 16885, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25224471999990783, "nonpadding_tokens": 9527, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25122831599992423, "nonpadding_tokens": 2181, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25523026900009427, "nonpadding_tokens": 2929, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036719600007018016, "prefetch": {"data_wait_seconds": 0.024669897999913815, "consumed_batches": 2633, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3183, "tokens": 230280709, "loss_tokens": 64155, "loss": 2.2665541462084016, "lr": 0.0002988139397637602, "optimizer_lrs": [0.01992092931758401, 0.0002988139397637602], "gradient_norm": 0.48315006494522095, "seconds": 0.30422739000005095, "nonpadding_tokens_per_second": 210878.44851835747, "ranks": [{"compute_seconds": 0.3008017160000236, "nonpadding_tokens": 4345, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2976968160000979, "nonpadding_tokens": 3407, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.300120250999953, "nonpadding_tokens": 3909, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3008067130001564, "nonpadding_tokens": 11349, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2984016049999809, "nonpadding_tokens": 2021, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2986346220000087, "nonpadding_tokens": 16267, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29589607399998386, "nonpadding_tokens": 10114, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3019080550000126, "nonpadding_tokens": 12743, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00032904199997574324, "prefetch": {"data_wait_seconds": 0.024677597999883005, "consumed_batches": 2634, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3184, "tokens": 230333592, "loss_tokens": 52883, "loss": 2.0681983104211183, "lr": 0.000298813344005372, "optimizer_lrs": [0.019920889600358135, 0.000298813344005372], "gradient_norm": 0.7177486419677734, "seconds": 0.29652847000011207, "nonpadding_tokens_per_second": 178340.3799303993, "ranks": [{"compute_seconds": 0.29243976299994756, "nonpadding_tokens": 6614, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29410435699992377, "nonpadding_tokens": 3172, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2904945539999062, "nonpadding_tokens": 3650, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2932269789998827, "nonpadding_tokens": 8794, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2940607230000296, "nonpadding_tokens": 14655, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2921296600000005, "nonpadding_tokens": 5625, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28931783100006214, "nonpadding_tokens": 7083, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2941539579999244, "nonpadding_tokens": 3290, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003931930000362627, "prefetch": {"data_wait_seconds": 0.024686708999709595, "consumed_batches": 2635, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3185, "tokens": 230428667, "loss_tokens": 95075, "loss": 2.6482946029450436, "lr": 0.00029881227255450286, "optimizer_lrs": [0.019920818170300194, 0.00029881227255450286], "gradient_norm": 1.5510810613632202, "seconds": 0.33349400499992043, "nonpadding_tokens_per_second": 285087.5835085032, "ranks": [{"compute_seconds": 0.33051647199999934, "nonpadding_tokens": 9197, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33012424399998963, "nonpadding_tokens": 5970, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32597134200000255, "nonpadding_tokens": 7775, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32698937200007094, "nonpadding_tokens": 10568, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32549650399982966, "nonpadding_tokens": 9535, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32838373699996737, "nonpadding_tokens": 9327, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3303157889999966, "nonpadding_tokens": 17763, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.331169311000167, "nonpadding_tokens": 24940, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003356609997808846, "prefetch": {"data_wait_seconds": 0.0246949709998745, "consumed_batches": 2636, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3186, "tokens": 230589507, "loss_tokens": 160840, "loss": 2.5830981099229047, "lr": 0.00029881045886681937, "optimizer_lrs": [0.01992069725778796, 0.00029881045886681937], "gradient_norm": 1.0005065202713013, "seconds": 0.30597588099999484, "nonpadding_tokens_per_second": 525662.3478763762, "ranks": [{"compute_seconds": 0.2730414229999951, "nonpadding_tokens": 9034, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28087695900012477, "nonpadding_tokens": 13540, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27752731700002187, "nonpadding_tokens": 24434, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27816627900006097, "nonpadding_tokens": 28571, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2767710010000428, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27782101700017847, "nonpadding_tokens": 18803, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2768741830000181, "nonpadding_tokens": 15703, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2816544940001222, "nonpadding_tokens": 17987, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.014312127999801305, "prefetch": {"data_wait_seconds": 0.024704354999769862, "consumed_batches": 2637, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3187, "tokens": 230667625, "loss_tokens": 78118, "loss": 2.130581500102409, "lr": 0.0002988095774841078, "optimizer_lrs": [0.019920638498940522, 0.0002988095774841078], "gradient_norm": 0.4720868766307831, "seconds": 0.2728983629999675, "nonpadding_tokens_per_second": 286253.0912287272, "ranks": [{"compute_seconds": 0.2683667220001098, "nonpadding_tokens": 13346, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2663019550000172, "nonpadding_tokens": 10501, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26904814899990015, "nonpadding_tokens": 4329, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26945798199994897, "nonpadding_tokens": 9761, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2701226240001233, "nonpadding_tokens": 21153, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2689310949999708, "nonpadding_tokens": 7701, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2699282169999151, "nonpadding_tokens": 4868, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27054162299987183, "nonpadding_tokens": 6459, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004064139998263272, "prefetch": {"data_wait_seconds": 0.02471504999967067, "consumed_batches": 2638, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3188, "tokens": 230730299, "loss_tokens": 62674, "loss": 2.122282565338099, "lr": 0.00029880887011662016, "optimizer_lrs": [0.01992059134110801, 0.00029880887011662016], "gradient_norm": 1.0501785278320312, "seconds": 0.27905554199992366, "nonpadding_tokens_per_second": 224593.28186364114, "ranks": [{"compute_seconds": 0.27318159700007527, "nonpadding_tokens": 2906, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27383741699986786, "nonpadding_tokens": 7681, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27115600799993445, "nonpadding_tokens": 9332, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2763884879998386, "nonpadding_tokens": 10431, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27676304500005244, "nonpadding_tokens": 11617, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2746249250001256, "nonpadding_tokens": 10005, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27048249800009216, "nonpadding_tokens": 2204, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.271550341999955, "nonpadding_tokens": 8498, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003378530000190949, "prefetch": {"data_wait_seconds": 0.024724151999592436, "consumed_batches": 2639, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3189, "tokens": 230804963, "loss_tokens": 74664, "loss": 1.9400271549876782, "lr": 0.00029880802715147537, "optimizer_lrs": [0.019920535143431693, 0.00029880802715147537], "gradient_norm": 0.550044059753418, "seconds": 0.3009237710000434, "nonpadding_tokens_per_second": 248115.99213938214, "ranks": [{"compute_seconds": 0.2959581420000177, "nonpadding_tokens": 8031, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2970307699999921, "nonpadding_tokens": 4524, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2967144650001501, "nonpadding_tokens": 8224, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29509815500000514, "nonpadding_tokens": 4422, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2977383689999442, "nonpadding_tokens": 13714, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2962075919999734, "nonpadding_tokens": 4638, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29265881400010585, "nonpadding_tokens": 21120, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2983091730000069, "nonpadding_tokens": 9991, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040752600011728646, "prefetch": {"data_wait_seconds": 0.02473274299968864, "consumed_batches": 2640, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3190, "tokens": 230845811, "loss_tokens": 40848, "loss": 2.264155183546318, "lr": 0.00029880756584722926, "optimizer_lrs": [0.019920504389815286, 0.00029880756584722926], "gradient_norm": 0.49268558621406555, "seconds": 0.2743770440001754, "nonpadding_tokens_per_second": 148875.42851425242, "ranks": [{"compute_seconds": 0.270249498000112, "nonpadding_tokens": 4719, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26804746700008764, "nonpadding_tokens": 3707, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26802156799999466, "nonpadding_tokens": 3153, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26857155900006546, "nonpadding_tokens": 5152, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27153805100010686, "nonpadding_tokens": 6368, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26988697100000536, "nonpadding_tokens": 3403, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2667443320001439, "nonpadding_tokens": 12158, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27197809900007996, "nonpadding_tokens": 2188, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003642510000645416, "prefetch": {"data_wait_seconds": 0.024741321999684374, "consumed_batches": 2641, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3191, "tokens": 230899806, "loss_tokens": 53995, "loss": 1.9920158984628207, "lr": 0.0002988069559350589, "optimizer_lrs": [0.01992046372900393, 0.0002988069559350589], "gradient_norm": 1.0706772804260254, "seconds": 0.23126506099993094, "nonpadding_tokens_per_second": 233476.68587091987, "ranks": [{"compute_seconds": 0.22853000700001758, "nonpadding_tokens": 2128, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22719540499997493, "nonpadding_tokens": 2489, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.227773230999901, "nonpadding_tokens": 14464, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2241995730000781, "nonpadding_tokens": 14833, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22889157999998133, "nonpadding_tokens": 5592, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22602440099990417, "nonpadding_tokens": 5733, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22876622800004043, "nonpadding_tokens": 1768, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2233884320000925, "nonpadding_tokens": 6988, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003481800001736701, "prefetch": {"data_wait_seconds": 0.02474966899963249, "consumed_batches": 2642, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3192, "tokens": 230973653, "loss_tokens": 73847, "loss": 2.258986908743754, "lr": 0.0002988061215289802, "optimizer_lrs": [0.019920408101932015, 0.0002988061215289802], "gradient_norm": 0.45909813046455383, "seconds": 0.2996369390000382, "nonpadding_tokens_per_second": 246454.9272410989, "ranks": [{"compute_seconds": 0.2958734070000446, "nonpadding_tokens": 5633, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29630560300006437, "nonpadding_tokens": 8744, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2936065429998962, "nonpadding_tokens": 7137, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29723307800009025, "nonpadding_tokens": 18263, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29234994500006906, "nonpadding_tokens": 5580, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29521390599984443, "nonpadding_tokens": 4295, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2972924440000497, "nonpadding_tokens": 13891, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29271855699994376, "nonpadding_tokens": 10304, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035875599996870733, "prefetch": {"data_wait_seconds": 0.024759691999818756, "consumed_batches": 2643, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3193, "tokens": 231021038, "loss_tokens": 47385, "loss": 2.0605391671942597, "lr": 0.00029880558596725353, "optimizer_lrs": [0.019920372397816903, 0.00029880558596725353], "gradient_norm": 0.4220854938030243, "seconds": 0.3125204479999866, "nonpadding_tokens_per_second": 151622.07882154975, "ranks": [{"compute_seconds": 0.3099997300000723, "nonpadding_tokens": 12377, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30653155300001345, "nonpadding_tokens": 2574, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3093560330000855, "nonpadding_tokens": 6651, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30990979199987123, "nonpadding_tokens": 5045, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3065552300001855, "nonpadding_tokens": 5675, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30899184999998397, "nonpadding_tokens": 2582, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.306068951999805, "nonpadding_tokens": 10142, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30713946500009115, "nonpadding_tokens": 2339, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037827799997103284, "prefetch": {"data_wait_seconds": 0.024767950999788013, "consumed_batches": 2644, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3194, "tokens": 231084990, "loss_tokens": 63952, "loss": 2.105725290061296, "lr": 0.0002988048629699651, "optimizer_lrs": [0.019920324197997673, 0.0002988048629699651], "gradient_norm": 0.5096362829208374, "seconds": 0.30809760200008895, "nonpadding_tokens_per_second": 207570.58667396422, "ranks": [{"compute_seconds": 0.30567273599990585, "nonpadding_tokens": 2975, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30162935000021207, "nonpadding_tokens": 4696, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3020389680000335, "nonpadding_tokens": 2483, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3052525630000673, "nonpadding_tokens": 6783, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30560252699979173, "nonpadding_tokens": 7925, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.302577102999976, "nonpadding_tokens": 7563, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.305015142000002, "nonpadding_tokens": 10889, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30199841900002866, "nonpadding_tokens": 20638, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044010999999954947, "prefetch": {"data_wait_seconds": 0.024777438999947776, "consumed_batches": 2645, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3195, "tokens": 231153441, "loss_tokens": 68451, "loss": 2.3525156224890797, "lr": 0.00029880408886877796, "optimizer_lrs": [0.019920272591251866, 0.00029880408886877796], "gradient_norm": 0.44027963280677795, "seconds": 0.3433390350000991, "nonpadding_tokens_per_second": 199368.53378754397, "ranks": [{"compute_seconds": 0.33793758900014836, "nonpadding_tokens": 20679, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34077130500008934, "nonpadding_tokens": 8038, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3377652599999692, "nonpadding_tokens": 7058, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3402247929998339, "nonpadding_tokens": 3582, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.34100528099997973, "nonpadding_tokens": 4286, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3392418359999283, "nonpadding_tokens": 3933, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3408062209998661, "nonpadding_tokens": 9460, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33700256499992065, "nonpadding_tokens": 11415, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033540100002937834, "prefetch": {"data_wait_seconds": 0.024786381999959417, "consumed_batches": 2646, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3196, "tokens": 231227258, "loss_tokens": 73817, "loss": 2.5685303266862647, "lr": 0.00029880325380476816, "optimizer_lrs": [0.01992021692031788, 0.00029880325380476816], "gradient_norm": 0.5304039716720581, "seconds": 0.2982498389999364, "nonpadding_tokens_per_second": 247500.55271619355, "ranks": [{"compute_seconds": 0.29361239800005023, "nonpadding_tokens": 10401, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29569982099997105, "nonpadding_tokens": 10099, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2914106880000418, "nonpadding_tokens": 7848, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29317853800012017, "nonpadding_tokens": 6340, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29269895900006304, "nonpadding_tokens": 9981, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2940359020001324, "nonpadding_tokens": 13544, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29092916599984164, "nonpadding_tokens": 5268, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2914371219999339, "nonpadding_tokens": 10336, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041212700011783454, "prefetch": {"data_wait_seconds": 0.02479667599978086, "consumed_batches": 2647, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3197, "tokens": 231283889, "loss_tokens": 56631, "loss": 2.2676980419293318, "lr": 0.00029880261296284045, "optimizer_lrs": [0.0199201741975227, 0.00029880261296284045], "gradient_norm": 0.44785046577453613, "seconds": 0.22555964599996514, "nonpadding_tokens_per_second": 251068.84588748091, "ranks": [{"compute_seconds": 0.19508738400008951, "nonpadding_tokens": 6471, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19318294200002128, "nonpadding_tokens": 3001, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.19287885800008553, "nonpadding_tokens": 7751, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.19330949200002578, "nonpadding_tokens": 4054, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.19670493000012357, "nonpadding_tokens": 17742, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1938985900001171, "nonpadding_tokens": 8152, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19613721899986558, "nonpadding_tokens": 4604, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.19200484699990739, "nonpadding_tokens": 4856, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00799712000002728, "prefetch": {"data_wait_seconds": 0.024806657999718595, "consumed_batches": 2648, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3198, "tokens": 231355960, "loss_tokens": 72071, "loss": 2.4765123107768727, "lr": 0.00029880179715340753, "optimizer_lrs": [0.01992011981022717, 0.00029880179715340753], "gradient_norm": 0.4122043550014496, "seconds": 0.2710748380000041, "nonpadding_tokens_per_second": 265871.2277824876, "ranks": [{"compute_seconds": 0.26792302099988774, "nonpadding_tokens": 2995, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26879047300008096, "nonpadding_tokens": 20391, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26773916299998746, "nonpadding_tokens": 5106, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26810317499985103, "nonpadding_tokens": 4581, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2647529939999913, "nonpadding_tokens": 10905, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26602932699984194, "nonpadding_tokens": 2308, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.264322201999903, "nonpadding_tokens": 13299, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26480380300017714, "nonpadding_tokens": 12486, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003567189999102993, "prefetch": {"data_wait_seconds": 0.024815565999645628, "consumed_batches": 2649, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3199, "tokens": 231423115, "loss_tokens": 67155, "loss": 1.8617524290819745, "lr": 0.00029880103674182905, "optimizer_lrs": [0.019920069116121937, 0.00029880103674182905], "gradient_norm": 0.4270090162754059, "seconds": 0.3180269330000556, "nonpadding_tokens_per_second": 211161.3609781542, "ranks": [{"compute_seconds": 0.3140535949999048, "nonpadding_tokens": 12975, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31506849100014733, "nonpadding_tokens": 8041, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31423835100008546, "nonpadding_tokens": 10452, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3147517979998611, "nonpadding_tokens": 2770, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3151394369999707, "nonpadding_tokens": 6918, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31379366100009065, "nonpadding_tokens": 18147, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31106563000003007, "nonpadding_tokens": 4253, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3157331580000573, "nonpadding_tokens": 3599, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036824800008616876, "prefetch": {"data_wait_seconds": 0.024824117999699524, "consumed_batches": 2650, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3200, "tokens": 231500734, "loss_tokens": 77619, "loss": 2.199810331555418, "lr": 0.0002988001575447837, "optimizer_lrs": [0.019920010502985582, 0.0002988001575447837], "gradient_norm": 0.4137149155139923, "seconds": 0.31467564700005823, "nonpadding_tokens_per_second": 246663.51126938537, "ranks": [{"compute_seconds": 0.25633623400017314, "nonpadding_tokens": 7915, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.258334356999967, "nonpadding_tokens": 6773, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25558550300002025, "nonpadding_tokens": 12104, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25786332500001663, "nonpadding_tokens": 6758, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2539489279999998, "nonpadding_tokens": 12399, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25712516900011906, "nonpadding_tokens": 16039, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2542719739999484, "nonpadding_tokens": 3600, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25871804299981704, "nonpadding_tokens": 12031, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.033058459999892875, "prefetch": {"data_wait_seconds": 0.024831976999806216, "consumed_batches": 2651, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3201, "tokens": 231541124, "loss_tokens": 40390, "loss": 2.2267327153998515, "lr": 0.0002987996999169037, "optimizer_lrs": [0.019919979994460248, 0.0002987996999169037], "gradient_norm": 0.4212777316570282, "seconds": 0.2955800300001101, "nonpadding_tokens_per_second": 136646.57926986797, "ranks": [{"compute_seconds": 0.25358494700003575, "nonpadding_tokens": 1847, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2707556719999502, "nonpadding_tokens": 14617, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27333331699992414, "nonpadding_tokens": 2487, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27196448399990913, "nonpadding_tokens": 3426, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2740305240001817, "nonpadding_tokens": 2572, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2733252790001188, "nonpadding_tokens": 5273, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2702650219998759, "nonpadding_tokens": 3778, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2710852289999366, "nonpadding_tokens": 6390, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004946400001699658, "prefetch": {"data_wait_seconds": 0.024842162999632933, "consumed_batches": 2652, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3202, "tokens": 231595243, "loss_tokens": 54119, "loss": 2.226971321301207, "lr": 0.0002987990866001611, "optimizer_lrs": [0.019919939106677408, 0.0002987990866001611], "gradient_norm": 0.4328575134277344, "seconds": 0.21470291800005725, "nonpadding_tokens_per_second": 252064.576038904, "ranks": [{"compute_seconds": 0.21197314100004405, "nonpadding_tokens": 3266, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2081620500000554, "nonpadding_tokens": 6321, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2076054159999785, "nonpadding_tokens": 8026, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20799002800004018, "nonpadding_tokens": 9280, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.20662811600004716, "nonpadding_tokens": 8948, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2091475999998238, "nonpadding_tokens": 3747, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2065029959999265, "nonpadding_tokens": 7289, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2123822519999976, "nonpadding_tokens": 7242, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003723320000972308, "prefetch": {"data_wait_seconds": 0.02485161399972924, "consumed_batches": 2653, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3203, "tokens": 231661723, "loss_tokens": 66480, "loss": 2.2723398954572804, "lr": 0.0002987983329859254, "optimizer_lrs": [0.019919888865728364, 0.0002987983329859254], "gradient_norm": 0.646263837814331, "seconds": 0.3546191020000151, "nonpadding_tokens_per_second": 187468.75062583957, "ranks": [{"compute_seconds": 0.3494666750000306, "nonpadding_tokens": 3852, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.34785113999987516, "nonpadding_tokens": 2943, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34893877399986195, "nonpadding_tokens": 9045, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3494917230000283, "nonpadding_tokens": 11906, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.34780063599987443, "nonpadding_tokens": 13250, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34880475099998876, "nonpadding_tokens": 9411, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3514583329999823, "nonpadding_tokens": 10194, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3522413860000597, "nonpadding_tokens": 5879, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003753680000500026, "prefetch": {"data_wait_seconds": 0.02486087999973563, "consumed_batches": 2654, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3204, "tokens": 231722174, "loss_tokens": 60451, "loss": 2.3072700410249625, "lr": 0.0002987976475118743, "optimizer_lrs": [0.01991984316745829, 0.0002987976475118743], "gradient_norm": 0.514271080493927, "seconds": 0.2857156989998657, "nonpadding_tokens_per_second": 211577.45343222603, "ranks": [{"compute_seconds": 0.21171831099991323, "nonpadding_tokens": 11842, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2263886000000639, "nonpadding_tokens": 9536, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22634723700002723, "nonpadding_tokens": 6477, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2241401239998595, "nonpadding_tokens": 1720, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22744986999987304, "nonpadding_tokens": 8359, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22551861700003428, "nonpadding_tokens": 6214, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22206887600009395, "nonpadding_tokens": 7852, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2233969009998873, "nonpadding_tokens": 8451, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.042387154000152805, "prefetch": {"data_wait_seconds": 0.024869360999630885, "consumed_batches": 2655, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3205, "tokens": 231805260, "loss_tokens": 83086, "loss": 2.46901064860506, "lr": 0.00029879670505450343, "optimizer_lrs": [0.019919780336966897, 0.00029879670505450343], "gradient_norm": 0.4588806927204132, "seconds": 0.347776113000009, "nonpadding_tokens_per_second": 238906.5749320105, "ranks": [{"compute_seconds": 0.34516346299983525, "nonpadding_tokens": 20936, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.344672355000057, "nonpadding_tokens": 6785, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3443722740000794, "nonpadding_tokens": 12421, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3446825880000688, "nonpadding_tokens": 5862, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.34539523299986286, "nonpadding_tokens": 13389, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34374028699994597, "nonpadding_tokens": 9345, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3451794669999799, "nonpadding_tokens": 8602, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34170952299996316, "nonpadding_tokens": 5746, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003572289999738132, "prefetch": {"data_wait_seconds": 0.024879045999753657, "consumed_batches": 2656, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3206, "tokens": 231847169, "loss_tokens": 41909, "loss": 2.241737119413491, "lr": 0.0002987962295347609, "optimizer_lrs": [0.01991974863565073, 0.0002987962295347609], "gradient_norm": 0.5934762358665466, "seconds": 0.23663783899996815, "nonpadding_tokens_per_second": 177101.8539431711, "ranks": [{"compute_seconds": 0.23202965299992684, "nonpadding_tokens": 2355, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23203746700005468, "nonpadding_tokens": 5626, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22910446300011245, "nonpadding_tokens": 7996, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22929933800014624, "nonpadding_tokens": 5017, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.233609677000004, "nonpadding_tokens": 3420, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23148468400017919, "nonpadding_tokens": 8403, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22797780800010514, "nonpadding_tokens": 4153, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23430097400000705, "nonpadding_tokens": 4939, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000382837999950425, "prefetch": {"data_wait_seconds": 0.02488708599976519, "consumed_batches": 2657, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3207, "tokens": 231917115, "loss_tokens": 69946, "loss": 2.5621919498613215, "lr": 0.00029879543568539155, "optimizer_lrs": [0.019919695712359436, 0.00029879543568539155], "gradient_norm": 0.3939652144908905, "seconds": 0.3371089990000655, "nonpadding_tokens_per_second": 207487.78646513203, "ranks": [{"compute_seconds": 0.33257391899996946, "nonpadding_tokens": 6923, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3337704279999798, "nonpadding_tokens": 6712, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3346449119999306, "nonpadding_tokens": 5551, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3347383580000951, "nonpadding_tokens": 19706, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3296099179999601, "nonpadding_tokens": 7138, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33264591699980883, "nonpadding_tokens": 6572, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3348062469999604, "nonpadding_tokens": 5283, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3301939770001354, "nonpadding_tokens": 12061, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003949769998143893, "prefetch": {"data_wait_seconds": 0.024895071999665674, "consumed_batches": 2658, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3208, "tokens": 231972175, "loss_tokens": 55060, "loss": 1.9401622945423174, "lr": 0.00029879481060085257, "optimizer_lrs": [0.019919654040056838, 0.00029879481060085257], "gradient_norm": 0.7008670568466187, "seconds": 0.33493892100000267, "nonpadding_tokens_per_second": 164388.181091679, "ranks": [{"compute_seconds": 0.3317101929999353, "nonpadding_tokens": 11294, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33249121399990145, "nonpadding_tokens": 12831, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32891170400012015, "nonpadding_tokens": 2166, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3316777469999579, "nonpadding_tokens": 12382, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33237701099983497, "nonpadding_tokens": 3985, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3308158119998552, "nonpadding_tokens": 3941, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32811657399997785, "nonpadding_tokens": 4955, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3287719840000136, "nonpadding_tokens": 3506, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0005098869999073941, "prefetch": {"data_wait_seconds": 0.024904550999693242, "consumed_batches": 2659, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3209, "tokens": 232031159, "loss_tokens": 58984, "loss": 2.5071211048758983, "lr": 0.0002987941407888871, "optimizer_lrs": [0.01991960938592581, 0.0002987941407888871], "gradient_norm": 0.5459513068199158, "seconds": 0.36043973100004223, "nonpadding_tokens_per_second": 163644.55670951848, "ranks": [{"compute_seconds": 0.3564756129999296, "nonpadding_tokens": 7368, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3573469020000175, "nonpadding_tokens": 5268, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3567428349999773, "nonpadding_tokens": 4388, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3570114819999617, "nonpadding_tokens": 22544, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.353182428999844, "nonpadding_tokens": 5840, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.35615031900010763, "nonpadding_tokens": 4520, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3528383140001097, "nonpadding_tokens": 4843, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35809820800000125, "nonpadding_tokens": 4213, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003758010000183276, "prefetch": {"data_wait_seconds": 0.024914376999689125, "consumed_batches": 2660, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3210, "tokens": 232078695, "loss_tokens": 47536, "loss": 2.2353269101312687, "lr": 0.0002987936008435993, "optimizer_lrs": [0.01991957338957329, 0.0002987936008435993], "gradient_norm": 0.43518924713134766, "seconds": 0.2458615269999882, "nonpadding_tokens_per_second": 193344.60572191223, "ranks": [{"compute_seconds": 0.24334169499979907, "nonpadding_tokens": 2818, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24258056400003625, "nonpadding_tokens": 8633, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24008774100002483, "nonpadding_tokens": 9542, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2432053420000102, "nonpadding_tokens": 6496, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23876227699997798, "nonpadding_tokens": 4157, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2419635119999839, "nonpadding_tokens": 5271, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24341648799986615, "nonpadding_tokens": 4403, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24013112500006173, "nonpadding_tokens": 6216, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041837899993879546, "prefetch": {"data_wait_seconds": 0.024924539999759077, "consumed_batches": 2661, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3211, "tokens": 232152794, "loss_tokens": 74099, "loss": 2.353501734166453, "lr": 0.0002987927589382635, "optimizer_lrs": [0.019919517262550902, 0.0002987927589382635], "gradient_norm": 0.37975645065307617, "seconds": 0.3290003570000408, "nonpadding_tokens_per_second": 225224.67961939264, "ranks": [{"compute_seconds": 0.32335401600016667, "nonpadding_tokens": 10729, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32600439500015455, "nonpadding_tokens": 11484, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32495191199996043, "nonpadding_tokens": 5100, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3233711950001634, "nonpadding_tokens": 4396, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32597920900002464, "nonpadding_tokens": 18426, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3249912230000973, "nonpadding_tokens": 4801, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.325351234999971, "nonpadding_tokens": 9406, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32666937499993765, "nonpadding_tokens": 9757, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037982000003466965, "prefetch": {"data_wait_seconds": 0.024935362999897848, "consumed_batches": 2662, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3212, "tokens": 232227740, "loss_tokens": 74946, "loss": 2.3835582452699278, "lr": 0.00029879190711202093, "optimizer_lrs": [0.01991946047413473, 0.00029879190711202093], "gradient_norm": 0.4223138689994812, "seconds": 0.2792388479999772, "nonpadding_tokens_per_second": 268393.88765851845, "ranks": [{"compute_seconds": 0.27395167099984974, "nonpadding_tokens": 12321, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2763132919999407, "nonpadding_tokens": 18911, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27521536000017477, "nonpadding_tokens": 4028, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27588782999987416, "nonpadding_tokens": 5269, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.276248373000044, "nonpadding_tokens": 19837, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2750482409999222, "nonpadding_tokens": 9421, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2722441980001804, "nonpadding_tokens": 2307, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2769111499999326, "nonpadding_tokens": 2852, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003957429998990847, "prefetch": {"data_wait_seconds": 0.024944862999930706, "consumed_batches": 2663, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3213, "tokens": 232286546, "loss_tokens": 58806, "loss": 2.316550660221746, "lr": 0.00029879123852147283, "optimizer_lrs": [0.019919415901431523, 0.00029879123852147283], "gradient_norm": 0.46004894375801086, "seconds": 0.26839010000003327, "nonpadding_tokens_per_second": 219106.44245071898, "ranks": [{"compute_seconds": 0.26563858200006507, "nonpadding_tokens": 3684, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26613409900005536, "nonpadding_tokens": 3223, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2646406669998669, "nonpadding_tokens": 6331, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2657061639999938, "nonpadding_tokens": 4528, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26579012500019417, "nonpadding_tokens": 12974, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26461916600010227, "nonpadding_tokens": 11538, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26226357799987454, "nonpadding_tokens": 1963, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26268260000006194, "nonpadding_tokens": 14565, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034557499998300045, "prefetch": {"data_wait_seconds": 0.024953997999958233, "consumed_batches": 2664, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3214, "tokens": 232367267, "loss_tokens": 80721, "loss": 2.3579590503090895, "lr": 0.0002987903204700874, "optimizer_lrs": [0.019919354698005827, 0.0002987903204700874], "gradient_norm": 0.5244580507278442, "seconds": 0.28152602199998, "nonpadding_tokens_per_second": 286726.60319835634, "ranks": [{"compute_seconds": 0.27925413499997376, "nonpadding_tokens": 12356, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2745089600000483, "nonpadding_tokens": 13940, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27708888099982687, "nonpadding_tokens": 1816, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2761349380000411, "nonpadding_tokens": 21897, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.278124637000019, "nonpadding_tokens": 19318, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2769990819999748, "nonpadding_tokens": 3626, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2738569630000711, "nonpadding_tokens": 1005, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2747977379999611, "nonpadding_tokens": 6763, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004033509999317175, "prefetch": {"data_wait_seconds": 0.024963029999938158, "consumed_batches": 2665, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3215, "tokens": 232426120, "loss_tokens": 58853, "loss": 2.3565743250131685, "lr": 0.0002987896509078885, "optimizer_lrs": [0.019919310060525903, 0.0002987896509078885], "gradient_norm": 0.42090103030204773, "seconds": 0.22086607899996125, "nonpadding_tokens_per_second": 266464.63896346133, "ranks": [{"compute_seconds": 0.21748664899996584, "nonpadding_tokens": 1691, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21815079400016657, "nonpadding_tokens": 3972, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21671522499991624, "nonpadding_tokens": 2578, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21561560700001792, "nonpadding_tokens": 18022, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2181623789999776, "nonpadding_tokens": 12813, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2151968250000209, "nonpadding_tokens": 3324, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21775120100005552, "nonpadding_tokens": 9625, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21848808999993707, "nonpadding_tokens": 6828, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037909399998170557, "prefetch": {"data_wait_seconds": 0.024972322000124336, "consumed_batches": 2666, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3216, "tokens": 232486009, "loss_tokens": 59889, "loss": 2.3248327113493295, "lr": 0.00029878896936995017, "optimizer_lrs": [0.019919264624663344, 0.00029878896936995017], "gradient_norm": 0.4600834250450134, "seconds": 0.28843008700005157, "nonpadding_tokens_per_second": 207637.839113467, "ranks": [{"compute_seconds": 0.2833763020000788, "nonpadding_tokens": 9677, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28456171799984986, "nonpadding_tokens": 8664, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28257933500003674, "nonpadding_tokens": 10905, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.285819575000005, "nonpadding_tokens": 13764, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2860697250000612, "nonpadding_tokens": 6418, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2832849129999886, "nonpadding_tokens": 3576, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28580916100008835, "nonpadding_tokens": 5071, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28195508199996766, "nonpadding_tokens": 1814, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039192300005197467, "prefetch": {"data_wait_seconds": 0.024980637000183492, "consumed_batches": 2667, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3217, "tokens": 232546298, "loss_tokens": 60289, "loss": 2.1254602829703595, "lr": 0.00029878828308712115, "optimizer_lrs": [0.019919218872474744, 0.00029878828308712115], "gradient_norm": 0.5092330574989319, "seconds": 0.339888363, "nonpadding_tokens_per_second": 177378.8295305656, "ranks": [{"compute_seconds": 0.33655992200010587, "nonpadding_tokens": 5756, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33351621100018747, "nonpadding_tokens": 15696, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33629036299998916, "nonpadding_tokens": 5100, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33456984999997985, "nonpadding_tokens": 7304, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3333729389999007, "nonpadding_tokens": 4965, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33422525100013445, "nonpadding_tokens": 6716, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33297445500011236, "nonpadding_tokens": 11705, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33754044500005875, "nonpadding_tokens": 3047, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036062999993191625, "prefetch": {"data_wait_seconds": 0.024998842000016452, "consumed_batches": 2668, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3218, "tokens": 232598434, "loss_tokens": 52136, "loss": 1.9700935414109253, "lr": 0.00029878768945563056, "optimizer_lrs": [0.01991917929704204, 0.00029878768945563056], "gradient_norm": 0.4195549488067627, "seconds": 0.2517636150000726, "nonpadding_tokens_per_second": 207083.14027022914, "ranks": [{"compute_seconds": 0.24811625500001355, "nonpadding_tokens": 3220, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2460767099998975, "nonpadding_tokens": 1930, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24619880399995964, "nonpadding_tokens": 12271, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24626348499987216, "nonpadding_tokens": 10215, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24897634800004198, "nonpadding_tokens": 13252, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24630295399992974, "nonpadding_tokens": 3112, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24898655199990571, "nonpadding_tokens": 4406, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24943782700006523, "nonpadding_tokens": 3730, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003810179998708918, "prefetch": {"data_wait_seconds": 0.025008882000022936, "consumed_batches": 2669, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3219, "tokens": 232635615, "loss_tokens": 37181, "loss": 2.1828804950781313, "lr": 0.0002987872660165176, "optimizer_lrs": [0.01991915106776784, 0.0002987872660165176], "gradient_norm": 0.42272108793258667, "seconds": 0.29010272200002873, "nonpadding_tokens_per_second": 128164.94703554115, "ranks": [{"compute_seconds": 0.26969890999998825, "nonpadding_tokens": 2415, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26795400699984384, "nonpadding_tokens": 5851, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.270603440000059, "nonpadding_tokens": 10181, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26740314600010606, "nonpadding_tokens": 2820, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26599428600002284, "nonpadding_tokens": 5358, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2684479989998181, "nonpadding_tokens": 3256, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.271247011000014, "nonpadding_tokens": 2594, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27229898900009175, "nonpadding_tokens": 4706, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003514030001952051, "prefetch": {"data_wait_seconds": 0.025016694999933975, "consumed_batches": 2670, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3220, "tokens": 232711662, "loss_tokens": 76047, "loss": 2.514954978500138, "lr": 0.0002987863997192951, "optimizer_lrs": [0.019919093314619673, 0.0002987863997192951], "gradient_norm": 0.44116175174713135, "seconds": 0.32938919799994437, "nonpadding_tokens_per_second": 230872.78047294327, "ranks": [{"compute_seconds": 0.32438428399996155, "nonpadding_tokens": 10360, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32531261999997696, "nonpadding_tokens": 7485, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3257737310000266, "nonpadding_tokens": 4553, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32616201899986663, "nonpadding_tokens": 13212, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32676225800014436, "nonpadding_tokens": 6837, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3236136929999702, "nonpadding_tokens": 6306, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3265417579998484, "nonpadding_tokens": 4597, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32703833400000804, "nonpadding_tokens": 22697, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038391099997170386, "prefetch": {"data_wait_seconds": 0.025025697000003788, "consumed_batches": 2671, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3221, "tokens": 232774514, "loss_tokens": 62852, "loss": 2.1282730660917712, "lr": 0.0002987856835018695, "optimizer_lrs": [0.0199190455667913, 0.0002987856835018695], "gradient_norm": 0.40393126010894775, "seconds": 0.24296511599982296, "nonpadding_tokens_per_second": 258687.34176656784, "ranks": [{"compute_seconds": 0.24055565699995896, "nonpadding_tokens": 5898, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2379803180001545, "nonpadding_tokens": 5798, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24063343800003167, "nonpadding_tokens": 14525, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2393407600000046, "nonpadding_tokens": 6295, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23754172200005996, "nonpadding_tokens": 4208, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2392628989998684, "nonpadding_tokens": 5382, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23803911400000288, "nonpadding_tokens": 1720, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23857084399992345, "nonpadding_tokens": 19026, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003992530000687111, "prefetch": {"data_wait_seconds": 0.025034115000110546, "consumed_batches": 2672, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3222, "tokens": 232870422, "loss_tokens": 95908, "loss": 2.0068685615381407, "lr": 0.0002987845901960543, "optimizer_lrs": [0.019918972679736954, 0.0002987845901960543], "gradient_norm": 0.42927631735801697, "seconds": 0.35346459000015784, "nonpadding_tokens_per_second": 271336.939295552, "ranks": [{"compute_seconds": 0.35097919899999397, "nonpadding_tokens": 15488, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35081128699994224, "nonpadding_tokens": 12451, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3496341840000241, "nonpadding_tokens": 26481, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35016731600012463, "nonpadding_tokens": 24660, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.35080285599997296, "nonpadding_tokens": 3688, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3495116510000571, "nonpadding_tokens": 7312, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34648617499988177, "nonpadding_tokens": 2756, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34702194500005135, "nonpadding_tokens": 3072, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040399499994236976, "prefetch": {"data_wait_seconds": 0.02504331099994772, "consumed_batches": 2673, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3223, "tokens": 232960241, "loss_tokens": 89819, "loss": 2.1675073620280787, "lr": 0.00029878356585787857, "optimizer_lrs": [0.01991890439052524, 0.00029878356585787857], "gradient_norm": 0.45478495955467224, "seconds": 0.28870424299998376, "nonpadding_tokens_per_second": 311110.77227917657, "ranks": [{"compute_seconds": 0.2851911329998984, "nonpadding_tokens": 5749, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28579031999993276, "nonpadding_tokens": 11840, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2846923639999659, "nonpadding_tokens": 18542, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28385454199997184, "nonpadding_tokens": 18527, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28242559699992853, "nonpadding_tokens": 6976, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28316461100007473, "nonpadding_tokens": 18712, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28198977700003525, "nonpadding_tokens": 5855, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.286332671000082, "nonpadding_tokens": 3618, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003883660001520184, "prefetch": {"data_wait_seconds": 0.025051431000022717, "consumed_batches": 2674, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3224, "tokens": 233019304, "loss_tokens": 59063, "loss": 2.8123510594619305, "lr": 0.000298782892041527, "optimizer_lrs": [0.019918859469435134, 0.000298782892041527], "gradient_norm": 0.5019267797470093, "seconds": 0.2664026959998864, "nonpadding_tokens_per_second": 221705.71426959275, "ranks": [{"compute_seconds": 0.2641645790001803, "nonpadding_tokens": 1232, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26342992499985485, "nonpadding_tokens": 4965, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2609260870001435, "nonpadding_tokens": 3578, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26349836700001106, "nonpadding_tokens": 15850, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25980063900010464, "nonpadding_tokens": 1934, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26062322799998583, "nonpadding_tokens": 3481, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25955340400014393, "nonpadding_tokens": 8947, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26407740900003773, "nonpadding_tokens": 19076, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035652300016408844, "prefetch": {"data_wait_seconds": 0.02506092399990223, "consumed_batches": 2675, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3225, "tokens": 233068348, "loss_tokens": 49044, "loss": 1.9896183783949106, "lr": 0.00029878233238513774, "optimizer_lrs": [0.019918822159009184, 0.00029878233238513774], "gradient_norm": 0.421265572309494, "seconds": 0.2910338530000445, "nonpadding_tokens_per_second": 168516.4783905483, "ranks": [{"compute_seconds": 0.27400916600004166, "nonpadding_tokens": 1366, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2739651120000417, "nonpadding_tokens": 3095, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2763602939999146, "nonpadding_tokens": 6493, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27672232200006874, "nonpadding_tokens": 5012, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27315497800009325, "nonpadding_tokens": 9413, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2743197710001368, "nonpadding_tokens": 3123, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2731111149998924, "nonpadding_tokens": 10598, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2747935500001404, "nonpadding_tokens": 9944, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039203799997267197, "prefetch": {"data_wait_seconds": 0.02508206099987831, "consumed_batches": 2676, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3226, "tokens": 233177091, "loss_tokens": 108743, "loss": 2.3190406508924712, "lr": 0.0002987810910280757, "optimizer_lrs": [0.019918739401871718, 0.0002987810910280757], "gradient_norm": 0.4322476387023926, "seconds": 0.28124317500009965, "nonpadding_tokens_per_second": 386651.1605124692, "ranks": [{"compute_seconds": 0.2778065100001186, "nonpadding_tokens": 10676, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2747467819999656, "nonpadding_tokens": 3235, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2774083689998861, "nonpadding_tokens": 18785, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2759754000001067, "nonpadding_tokens": 21130, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2742095920000338, "nonpadding_tokens": 18398, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2758982579998701, "nonpadding_tokens": 17214, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2778922640000019, "nonpadding_tokens": 2876, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27895755600002303, "nonpadding_tokens": 16429, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003462109998508822, "prefetch": {"data_wait_seconds": 0.025090481999725966, "consumed_batches": 2677, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3227, "tokens": 233242488, "loss_tokens": 65397, "loss": 2.162347374497301, "lr": 0.00029878034418465946, "optimizer_lrs": [0.019918689612310633, 0.00029878034418465946], "gradient_norm": 0.4403717815876007, "seconds": 0.3372753940000166, "nonpadding_tokens_per_second": 193897.92781621293, "ranks": [{"compute_seconds": 0.33386252399986915, "nonpadding_tokens": 13363, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33499284499998794, "nonpadding_tokens": 7040, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3334004329999516, "nonpadding_tokens": 2599, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3322062709999045, "nonpadding_tokens": 5114, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3347295659998508, "nonpadding_tokens": 9860, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3317792590000863, "nonpadding_tokens": 9782, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33067433900009746, "nonpadding_tokens": 12726, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33113875399999415, "nonpadding_tokens": 4913, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003576770000108809, "prefetch": {"data_wait_seconds": 0.025098740999695224, "consumed_batches": 2678, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3228, "tokens": 233298338, "loss_tokens": 55850, "loss": 2.2251219785138763, "lr": 0.00029877970618913657, "optimizer_lrs": [0.01991864707927577, 0.00029877970618913657], "gradient_norm": 0.41700536012649536, "seconds": 0.3146515720000025, "nonpadding_tokens_per_second": 177497.92141511868, "ranks": [{"compute_seconds": 0.30999768199990285, "nonpadding_tokens": 8085, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30877800500002195, "nonpadding_tokens": 8890, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3092789840000023, "nonpadding_tokens": 7749, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30956282900001497, "nonpadding_tokens": 4605, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3090164050001931, "nonpadding_tokens": 3769, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3114270309999938, "nonpadding_tokens": 5885, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3123266039999635, "nonpadding_tokens": 5727, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3092233109998688, "nonpadding_tokens": 11140, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036849000002803223, "prefetch": {"data_wait_seconds": 0.025107894999749192, "consumed_batches": 2679, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3229, "tokens": 233370689, "loss_tokens": 72351, "loss": 2.3282275815123494, "lr": 0.0002987788794495851, "optimizer_lrs": [0.019918591963305674, 0.0002987788794495851], "gradient_norm": 0.5149266719818115, "seconds": 0.3119792640000014, "nonpadding_tokens_per_second": 231909.64384094346, "ranks": [{"compute_seconds": 0.30166273000008914, "nonpadding_tokens": 8337, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3043265359999623, "nonpadding_tokens": 2501, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30472225700009403, "nonpadding_tokens": 5497, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.307153900000003, "nonpadding_tokens": 2038, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.307790137999973, "nonpadding_tokens": 16166, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30623968999998397, "nonpadding_tokens": 9573, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30767709199994897, "nonpadding_tokens": 22820, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3082225059999928, "nonpadding_tokens": 5419, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003401399999347632, "prefetch": {"data_wait_seconds": 0.02511608699974488, "consumed_batches": 2680, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3230, "tokens": 233432466, "loss_tokens": 61777, "loss": 1.913622914879324, "lr": 0.0002987781733163225, "optimizer_lrs": [0.019918544887754835, 0.0002987781733163225], "gradient_norm": 0.6675834655761719, "seconds": 0.3438043240000752, "nonpadding_tokens_per_second": 179686.5126105467, "ranks": [{"compute_seconds": 0.34039774300003955, "nonpadding_tokens": 2412, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33779743199988843, "nonpadding_tokens": 10905, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34015065400012645, "nonpadding_tokens": 3318, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33838737800010676, "nonpadding_tokens": 18516, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3414082540000436, "nonpadding_tokens": 7516, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33848990199999207, "nonpadding_tokens": 3369, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33732485600012296, "nonpadding_tokens": 2134, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33782481900016137, "nonpadding_tokens": 13607, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041125300003841403, "prefetch": {"data_wait_seconds": 0.025124364999740578, "consumed_batches": 2681, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3231, "tokens": 233499995, "loss_tokens": 67529, "loss": 2.3291662193279925, "lr": 0.0002987774012032003, "optimizer_lrs": [0.01991849341354669, 0.0002987774012032003], "gradient_norm": 0.4192928075790405, "seconds": 0.35029438400010804, "nonpadding_tokens_per_second": 192777.85509681245, "ranks": [{"compute_seconds": 0.34783751500003746, "nonpadding_tokens": 11220, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3475265729998682, "nonpadding_tokens": 7185, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34630094699991787, "nonpadding_tokens": 5411, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.34529080799984513, "nonpadding_tokens": 10068, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3432258070001808, "nonpadding_tokens": 5585, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3461895519999416, "nonpadding_tokens": 9310, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3428014499997971, "nonpadding_tokens": 15551, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34793418300000667, "nonpadding_tokens": 3199, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000350966000041808, "prefetch": {"data_wait_seconds": 0.025132539999688674, "consumed_batches": 2682, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3232, "tokens": 233583490, "loss_tokens": 83495, "loss": 2.209762373495419, "lr": 0.00029877644620235696, "optimizer_lrs": [0.0199184297468238, 0.00029877644620235696], "gradient_norm": 0.4387339949607849, "seconds": 0.2747487799999817, "nonpadding_tokens_per_second": 303895.7989185814, "ranks": [{"compute_seconds": 0.26192627299997184, "nonpadding_tokens": 2348, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27162200199995823, "nonpadding_tokens": 8218, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26943139699983476, "nonpadding_tokens": 4473, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2692426839998916, "nonpadding_tokens": 21782, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2721026530000472, "nonpadding_tokens": 5313, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27053998700012016, "nonpadding_tokens": 24968, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2715787969998473, "nonpadding_tokens": 6178, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2723905290001767, "nonpadding_tokens": 10215, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039016599998831225, "prefetch": {"data_wait_seconds": 0.02514109399972142, "consumed_batches": 2683, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3233, "tokens": 233645852, "loss_tokens": 62362, "loss": 2.407570415477374, "lr": 0.00029877573267485534, "optimizer_lrs": [0.01991838217832369, 0.00029877573267485534], "gradient_norm": 0.5992734432220459, "seconds": 0.3036625280001317, "nonpadding_tokens_per_second": 205366.13592301038, "ranks": [{"compute_seconds": 0.2998903190000419, "nonpadding_tokens": 2563, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3014637720000337, "nonpadding_tokens": 6932, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29865115600000536, "nonpadding_tokens": 6596, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.300869584000111, "nonpadding_tokens": 13486, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3013300719999279, "nonpadding_tokens": 5343, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2985446200000297, "nonpadding_tokens": 16675, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29734494099989206, "nonpadding_tokens": 5570, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29804052400004366, "nonpadding_tokens": 5197, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036326300005384837, "prefetch": {"data_wait_seconds": 0.02514998699957971, "consumed_batches": 2684, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3234, "tokens": 233736990, "loss_tokens": 91138, "loss": 2.369230246988084, "lr": 0.0002987746895285111, "optimizer_lrs": [0.019918312635234074, 0.0002987746895285111], "gradient_norm": 0.4224381148815155, "seconds": 0.26518936399997983, "nonpadding_tokens_per_second": 343671.40003400337, "ranks": [{"compute_seconds": 0.2629750090000016, "nonpadding_tokens": 7271, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2593124180000359, "nonpadding_tokens": 17116, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2617577500000152, "nonpadding_tokens": 17551, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26102755900001284, "nonpadding_tokens": 8378, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25930950999986635, "nonpadding_tokens": 6645, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26042557300002045, "nonpadding_tokens": 10521, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25900509500002045, "nonpadding_tokens": 3185, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2593284609999955, "nonpadding_tokens": 20471, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003429040000355599, "prefetch": {"data_wait_seconds": 0.025158523999564864, "consumed_batches": 2685, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3235, "tokens": 233830507, "loss_tokens": 93517, "loss": 2.285812171851107, "lr": 0.00029877361869297414, "optimizer_lrs": [0.019918241246198277, 0.00029877361869297414], "gradient_norm": 0.3969953954219818, "seconds": 0.3249336390001645, "nonpadding_tokens_per_second": 287803.38129273424, "ranks": [{"compute_seconds": 0.32199032799985616, "nonpadding_tokens": 4596, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3221398810001119, "nonpadding_tokens": 1845, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31919384299999365, "nonpadding_tokens": 5043, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3216393989998778, "nonpadding_tokens": 8599, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32259802600015064, "nonpadding_tokens": 4168, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31979780200003916, "nonpadding_tokens": 24680, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3221248370000467, "nonpadding_tokens": 27678, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3190250309999101, "nonpadding_tokens": 16908, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040061400000013236, "prefetch": {"data_wait_seconds": 0.025167273999571194, "consumed_batches": 2686, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3236, "tokens": 233902192, "loss_tokens": 71685, "loss": 2.283669875148218, "lr": 0.0002987727975339904, "optimizer_lrs": [0.01991818650226603, 0.0002987727975339904], "gradient_norm": 0.4110945463180542, "seconds": 0.3254662829999688, "nonpadding_tokens_per_second": 220253.22973319134, "ranks": [{"compute_seconds": 0.32257474900006855, "nonpadding_tokens": 18926, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32284473100003197, "nonpadding_tokens": 17092, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32165412300014395, "nonpadding_tokens": 7515, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.320216475000052, "nonpadding_tokens": 6024, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3225134250001247, "nonpadding_tokens": 4626, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31994784700009404, "nonpadding_tokens": 5436, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3186688089999734, "nonpadding_tokens": 6351, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32318476900013593, "nonpadding_tokens": 5715, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003447239998877194, "prefetch": {"data_wait_seconds": 0.02517542599953515, "consumed_batches": 2687, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3237, "tokens": 233985589, "loss_tokens": 83397, "loss": 2.2693211761214434, "lr": 0.0002987718418685894, "optimizer_lrs": [0.019918122791239296, 0.0002987718418685894], "gradient_norm": 0.8265330195426941, "seconds": 0.38763380499995037, "nonpadding_tokens_per_second": 215143.77467674852, "ranks": [{"compute_seconds": 0.38498082800015254, "nonpadding_tokens": 17137, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.385375440999951, "nonpadding_tokens": 10307, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.38247560300010264, "nonpadding_tokens": 8353, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.38488897199999883, "nonpadding_tokens": 10293, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.38106945999993513, "nonpadding_tokens": 12877, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38242217900005926, "nonpadding_tokens": 10790, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.38151415600009386, "nonpadding_tokens": 9111, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3819138680000833, "nonpadding_tokens": 4529, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000370236999970075, "prefetch": {"data_wait_seconds": 0.025184617999642, "consumed_batches": 2688, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3238, "tokens": 234046681, "loss_tokens": 61092, "loss": 2.028901211492503, "lr": 0.00029877114156635204, "optimizer_lrs": [0.01991807610442347, 0.00029877114156635204], "gradient_norm": 0.7140554189682007, "seconds": 0.26213255400011803, "nonpadding_tokens_per_second": 233057.66135392894, "ranks": [{"compute_seconds": 0.2597015719998126, "nonpadding_tokens": 3420, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.259432095999955, "nonpadding_tokens": 6821, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25856586400004744, "nonpadding_tokens": 1898, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25910564800005886, "nonpadding_tokens": 3545, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2551252650000606, "nonpadding_tokens": 12074, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25797610899985557, "nonpadding_tokens": 25510, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2590919419999409, "nonpadding_tokens": 3348, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25591318700003285, "nonpadding_tokens": 4476, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004099889999906736, "prefetch": {"data_wait_seconds": 0.025192854999616543, "consumed_batches": 2689, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3239, "tokens": 234123733, "loss_tokens": 77052, "loss": 2.0669551893526448, "lr": 0.0002987702580300942, "optimizer_lrs": [0.019918017202006283, 0.0002987702580300942], "gradient_norm": 0.5046907067298889, "seconds": 0.2807626190001429, "nonpadding_tokens_per_second": 274438.24350406416, "ranks": [{"compute_seconds": 0.27676803800000016, "nonpadding_tokens": 3423, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27851943500013476, "nonpadding_tokens": 15228, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2757684919999974, "nonpadding_tokens": 4358, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2764358760000505, "nonpadding_tokens": 8228, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27482188499993754, "nonpadding_tokens": 3628, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2775037290000455, "nonpadding_tokens": 15807, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27671985799997856, "nonpadding_tokens": 10799, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2753630540000813, "nonpadding_tokens": 15581, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036377599985826237, "prefetch": {"data_wait_seconds": 0.02520275499973934, "consumed_batches": 2690, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3240, "tokens": 234177001, "loss_tokens": 53268, "loss": 2.486684958605542, "lr": 0.0002987696470343143, "optimizer_lrs": [0.01991797646895429, 0.0002987696470343143], "gradient_norm": 0.464480459690094, "seconds": 0.27035311100007675, "nonpadding_tokens_per_second": 197031.2078265113, "ranks": [{"compute_seconds": 0.2660440470001504, "nonpadding_tokens": 5394, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2671966860000339, "nonpadding_tokens": 5355, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.264910799000063, "nonpadding_tokens": 2751, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26705497700004344, "nonpadding_tokens": 17576, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.268066374, "nonpadding_tokens": 10739, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2652569389999826, "nonpadding_tokens": 1309, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2672252979998575, "nonpadding_tokens": 7884, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2644832859998587, "nonpadding_tokens": 2260, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035049999996772385, "prefetch": {"data_wait_seconds": 0.025211078999703318, "consumed_batches": 2691, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3241, "tokens": 234268089, "loss_tokens": 91088, "loss": 2.0465065376339364, "lr": 0.0002987686018847455, "optimizer_lrs": [0.019917906792316368, 0.0002987686018847455], "gradient_norm": 0.4390714764595032, "seconds": 0.29517054899997675, "nonpadding_tokens_per_second": 308594.47295335407, "ranks": [{"compute_seconds": 0.2897835960000066, "nonpadding_tokens": 1830, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28852499300001, "nonpadding_tokens": 26817, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29117688500014083, "nonpadding_tokens": 18223, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28992347600001267, "nonpadding_tokens": 10156, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29211574999999357, "nonpadding_tokens": 10667, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2908587079998597, "nonpadding_tokens": 10371, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2878461430000243, "nonpadding_tokens": 3576, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2928898960001334, "nonpadding_tokens": 9448, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037389400017673324, "prefetch": {"data_wait_seconds": 0.025226315999773874, "consumed_batches": 2692, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3242, "tokens": 234314089, "loss_tokens": 46000, "loss": 2.218622282608696, "lr": 0.000298768073909937, "optimizer_lrs": [0.019917871593995804, 0.000298768073909937], "gradient_norm": 0.38700976967811584, "seconds": 0.30245096299995566, "nonpadding_tokens_per_second": 152090.77049626302, "ranks": [{"compute_seconds": 0.29559422499983157, "nonpadding_tokens": 2165, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3000637759998881, "nonpadding_tokens": 7620, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29920496700015065, "nonpadding_tokens": 5598, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29942113100014467, "nonpadding_tokens": 10799, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3001564099999996, "nonpadding_tokens": 4065, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29732412800012753, "nonpadding_tokens": 3261, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3000653879998936, "nonpadding_tokens": 8499, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29669054400005734, "nonpadding_tokens": 3993, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033970899994528736, "prefetch": {"data_wait_seconds": 0.025234257999912302, "consumed_batches": 2693, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3243, "tokens": 234377068, "loss_tokens": 62979, "loss": 2.50236859707204, "lr": 0.0002987673508723512, "optimizer_lrs": [0.01991782339149008, 0.0002987673508723512], "gradient_norm": 0.4602624177932739, "seconds": 0.3987156559999221, "nonpadding_tokens_per_second": 157954.67033281559, "ranks": [{"compute_seconds": 0.3939723089999916, "nonpadding_tokens": 5078, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.39236069300000054, "nonpadding_tokens": 12202, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.39517721999982314, "nonpadding_tokens": 7376, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.39544213299996045, "nonpadding_tokens": 2789, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3962639799999579, "nonpadding_tokens": 13705, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3946962100001201, "nonpadding_tokens": 6474, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3914360679998481, "nonpadding_tokens": 9845, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.39223321099984787, "nonpadding_tokens": 5510, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004366729999674135, "prefetch": {"data_wait_seconds": 0.025242529999971453, "consumed_batches": 2694, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3244, "tokens": 234433367, "loss_tokens": 56299, "loss": 2.205139495150891, "lr": 0.00029876670434652626, "optimizer_lrs": [0.019917780289768417, 0.00029876670434652626], "gradient_norm": 0.43303778767585754, "seconds": 0.3727762700000312, "nonpadding_tokens_per_second": 151026.24424026583, "ranks": [{"compute_seconds": 0.36757215700004053, "nonpadding_tokens": 2658, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3661929059999238, "nonpadding_tokens": 3078, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3690993919999528, "nonpadding_tokens": 9566, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.36746107699991626, "nonpadding_tokens": 10290, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.36613163800006987, "nonpadding_tokens": 11302, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.36803968900017026, "nonpadding_tokens": 12154, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36938135399987004, "nonpadding_tokens": 3246, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3704373969999324, "nonpadding_tokens": 4005, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038630599988209724, "prefetch": {"data_wait_seconds": 0.025251135999909025, "consumed_batches": 2695, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3245, "tokens": 234503303, "loss_tokens": 69936, "loss": 2.3362527435798444, "lr": 0.00029876590098128897, "optimizer_lrs": [0.019917726732085934, 0.00029876590098128897], "gradient_norm": 0.4033745527267456, "seconds": 0.3332920060001925, "nonpadding_tokens_per_second": 209834.015640806, "ranks": [{"compute_seconds": 0.33075529999996434, "nonpadding_tokens": 15726, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3304438959999061, "nonpadding_tokens": 3373, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32639372099993125, "nonpadding_tokens": 4933, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3272995690001608, "nonpadding_tokens": 7983, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33022759899995435, "nonpadding_tokens": 4884, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32888956900001176, "nonpadding_tokens": 5408, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32515369699990515, "nonpadding_tokens": 3923, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32629093300010936, "nonpadding_tokens": 23706, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037200300016593246, "prefetch": {"data_wait_seconds": 0.025277421999774674, "consumed_batches": 2696, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3246, "tokens": 234580600, "loss_tokens": 77297, "loss": 2.00303921077144, "lr": 0.00029876501275623215, "optimizer_lrs": [0.019917667517082144, 0.00029876501275623215], "gradient_norm": 0.6015479564666748, "seconds": 0.2969814329999281, "nonpadding_tokens_per_second": 260275.5304235424, "ranks": [{"compute_seconds": 0.2914186430000427, "nonpadding_tokens": 2386, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29411718099981954, "nonpadding_tokens": 11599, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29267505299981167, "nonpadding_tokens": 11809, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29088810099983675, "nonpadding_tokens": 7311, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29368784700000106, "nonpadding_tokens": 12090, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29239418799988925, "nonpadding_tokens": 10931, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28979807700011406, "nonpadding_tokens": 10979, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29442876799998885, "nonpadding_tokens": 10192, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004943050000747462, "prefetch": {"data_wait_seconds": 0.025288861999754886, "consumed_batches": 2697, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3247, "tokens": 234636885, "loss_tokens": 56285, "loss": 2.640674691303189, "lr": 0.00029876436578136714, "optimizer_lrs": [0.01991762438542448, 0.00029876436578136714], "gradient_norm": 0.4863477349281311, "seconds": 0.18836756999985482, "nonpadding_tokens_per_second": 298804.0881986394, "ranks": [{"compute_seconds": 0.18546533699986867, "nonpadding_tokens": 7716, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.1819492419999733, "nonpadding_tokens": 11047, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.18142897300003824, "nonpadding_tokens": 4111, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.18219416900001306, "nonpadding_tokens": 6790, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.18552865200013002, "nonpadding_tokens": 11908, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.18385327800001505, "nonpadding_tokens": 3787, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.18102573700002722, "nonpadding_tokens": 6612, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.18607713999995212, "nonpadding_tokens": 4314, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040294999985235336, "prefetch": {"data_wait_seconds": 0.025298765999835382, "consumed_batches": 2698, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3248, "tokens": 234729014, "loss_tokens": 92129, "loss": 2.149917167775619, "lr": 0.0002987633064292956, "optimizer_lrs": [0.01991755376195304, 0.0002987633064292956], "gradient_norm": 0.39441871643066406, "seconds": 0.30860436300008587, "nonpadding_tokens_per_second": 298534.34055296995, "ranks": [{"compute_seconds": 0.3051761379999789, "nonpadding_tokens": 14627, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3061562210000375, "nonpadding_tokens": 16435, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.302159778000032, "nonpadding_tokens": 14396, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3028272490000745, "nonpadding_tokens": 11021, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30142461199989157, "nonpadding_tokens": 9060, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3041001110000252, "nonpadding_tokens": 5643, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3016197029999148, "nonpadding_tokens": 8718, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30633415500005867, "nonpadding_tokens": 12229, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035126500006299466, "prefetch": {"data_wait_seconds": 0.025306912999852216, "consumed_batches": 2699, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3249, "tokens": 234816552, "loss_tokens": 87538, "loss": 2.3404735657657247, "lr": 0.0002987622994485613, "optimizer_lrs": [0.01991748662990409, 0.0002987622994485613], "gradient_norm": 0.5142253637313843, "seconds": 0.3381009710001308, "nonpadding_tokens_per_second": 258910.82105163825, "ranks": [{"compute_seconds": 0.318301646000009, "nonpadding_tokens": 18733, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3296963669999968, "nonpadding_tokens": 14972, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3272026720001122, "nonpadding_tokens": 9386, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32780989700017926, "nonpadding_tokens": 11494, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3303948360000959, "nonpadding_tokens": 10537, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3289272129998153, "nonpadding_tokens": 14107, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32985408100012137, "nonpadding_tokens": 1943, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3267699350001294, "nonpadding_tokens": 6366, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004257179998603533, "prefetch": {"data_wait_seconds": 0.025316277999721137, "consumed_batches": 2700, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3250, "tokens": 234880829, "loss_tokens": 64277, "loss": 2.0445921558255673, "lr": 0.0002987615597876535, "optimizer_lrs": [0.0199174373191769, 0.0002987615597876535], "gradient_norm": 0.4564257860183716, "seconds": 0.3186900529999548, "nonpadding_tokens_per_second": 201691.2652118738, "ranks": [{"compute_seconds": 0.3161763560001418, "nonpadding_tokens": 4006, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3142281870000261, "nonpadding_tokens": 8535, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31086121399994227, "nonpadding_tokens": 11018, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3121036610000374, "nonpadding_tokens": 14525, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3161669339999662, "nonpadding_tokens": 3930, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31313195999996424, "nonpadding_tokens": 6956, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31033204799996383, "nonpadding_tokens": 3625, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3110366720000002, "nonpadding_tokens": 11682, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035693099994205113, "prefetch": {"data_wait_seconds": 0.025326792999706527, "consumed_batches": 2701, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3251, "tokens": 234908557, "loss_tokens": 27728, "loss": 2.536650959769908, "lr": 0.0002987612406426702, "optimizer_lrs": [0.019917416042844683, 0.0002987612406426702], "gradient_norm": 0.4872627556324005, "seconds": 0.27945959699991363, "nonpadding_tokens_per_second": 99220.0672214115, "ranks": [{"compute_seconds": 0.27505567000002884, "nonpadding_tokens": 8674, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2745789269999932, "nonpadding_tokens": 2870, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27633459899993795, "nonpadding_tokens": 4304, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2770059819999915, "nonpadding_tokens": 1167, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27035189899993384, "nonpadding_tokens": 3707, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2741884659999414, "nonpadding_tokens": 1747, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27049152400013554, "nonpadding_tokens": 1800, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27164655100000346, "nonpadding_tokens": 3459, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042855800006691425, "prefetch": {"data_wait_seconds": 0.02533663599975, "consumed_batches": 2702, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3252, "tokens": 234970750, "loss_tokens": 62193, "loss": 2.245929756966218, "lr": 0.0002987605246620294, "optimizer_lrs": [0.019917368310801962, 0.0002987605246620294], "gradient_norm": 0.6759747862815857, "seconds": 0.25964955299991743, "nonpadding_tokens_per_second": 239526.69774101162, "ranks": [{"compute_seconds": 0.2554234259998793, "nonpadding_tokens": 3933, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25738379099993836, "nonpadding_tokens": 3326, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.255316811000057, "nonpadding_tokens": 12773, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.254941522000081, "nonpadding_tokens": 3509, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2536697949999507, "nonpadding_tokens": 6039, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.256039439999995, "nonpadding_tokens": 4314, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2534065390000251, "nonpadding_tokens": 17978, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25388221699995484, "nonpadding_tokens": 10321, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003892039999300323, "prefetch": {"data_wait_seconds": 0.025346236999851612, "consumed_batches": 2703, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3253, "tokens": 235042966, "loss_tokens": 72216, "loss": 2.2488169173036447, "lr": 0.0002987596930358781, "optimizer_lrs": [0.01991731286905854, 0.0002987596930358781], "gradient_norm": 0.41978389024734497, "seconds": 0.2653495819999989, "nonpadding_tokens_per_second": 272154.18790446897, "ranks": [{"compute_seconds": 0.2583213939999496, "nonpadding_tokens": 1505, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26094167599990215, "nonpadding_tokens": 7386, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.258189141999992, "nonpadding_tokens": 2664, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25886137599991343, "nonpadding_tokens": 6965, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2629184099998838, "nonpadding_tokens": 5615, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26052892800021255, "nonpadding_tokens": 18960, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25729199700003846, "nonpadding_tokens": 17765, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2583423359999415, "nonpadding_tokens": 11356, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003651919998901576, "prefetch": {"data_wait_seconds": 0.025354966999842077, "consumed_batches": 2704, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3254, "tokens": 235119356, "loss_tokens": 76390, "loss": 2.4954638614347426, "lr": 0.0002987588130405711, "optimizer_lrs": [0.01991725420270474, 0.0002987588130405711], "gradient_norm": 0.4131808578968048, "seconds": 0.26269865199992637, "nonpadding_tokens_per_second": 290789.46320600616, "ranks": [{"compute_seconds": 0.25446134300000267, "nonpadding_tokens": 7964, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2545163880001837, "nonpadding_tokens": 3394, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25745168899993587, "nonpadding_tokens": 3340, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25375296799984426, "nonpadding_tokens": 9893, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2523128780001116, "nonpadding_tokens": 13299, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25472588499997073, "nonpadding_tokens": 14764, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25194445099987206, "nonpadding_tokens": 5614, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.260229881999976, "nonpadding_tokens": 18122, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003317779999179038, "prefetch": {"data_wait_seconds": 0.025363729999710927, "consumed_batches": 2705, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3255, "tokens": 235190125, "loss_tokens": 70769, "loss": 2.3737253864686516, "lr": 0.000298757997520716, "optimizer_lrs": [0.019917199834714402, 0.000298757997520716], "gradient_norm": 0.5783607959747314, "seconds": 0.2547351079999771, "nonpadding_tokens_per_second": 277814.08128480805, "ranks": [{"compute_seconds": 0.2524085230002129, "nonpadding_tokens": 8274, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25148904399998173, "nonpadding_tokens": 9387, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24961858999995457, "nonpadding_tokens": 14485, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24877734400001827, "nonpadding_tokens": 3608, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24696499800006677, "nonpadding_tokens": 11410, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25036925599988535, "nonpadding_tokens": 13222, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24712692200000674, "nonpadding_tokens": 3660, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24836409400018056, "nonpadding_tokens": 6723, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00032946500004982227, "prefetch": {"data_wait_seconds": 0.025372328999722527, "consumed_batches": 2706, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3256, "tokens": 235237037, "loss_tokens": 46912, "loss": 2.090253759699011, "lr": 0.0002987574567745447, "optimizer_lrs": [0.019917163784969646, 0.0002987574567745447], "gradient_norm": 0.4403853416442871, "seconds": 0.23531205700010105, "nonpadding_tokens_per_second": 199360.80028393894, "ranks": [{"compute_seconds": 0.22838089800006856, "nonpadding_tokens": 4506, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2304034359999605, "nonpadding_tokens": 4209, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.232809503999988, "nonpadding_tokens": 5545, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22812991099999635, "nonpadding_tokens": 3092, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22700805200020113, "nonpadding_tokens": 14176, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2314006500000687, "nonpadding_tokens": 9458, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2267944660000012, "nonpadding_tokens": 2856, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2277242780000961, "nonpadding_tokens": 3070, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004186740000022837, "prefetch": {"data_wait_seconds": 0.02538180299980297, "consumed_batches": 2707, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3257, "tokens": 235292942, "loss_tokens": 55905, "loss": 2.385056680976657, "lr": 0.0002987568122146993, "optimizer_lrs": [0.01991712081431329, 0.0002987568122146993], "gradient_norm": 0.3818857669830322, "seconds": 0.32132352899998295, "nonpadding_tokens_per_second": 173983.5242504229, "ranks": [{"compute_seconds": 0.3171578829999362, "nonpadding_tokens": 2853, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3163652859998365, "nonpadding_tokens": 3661, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.313551390999919, "nonpadding_tokens": 7207, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3189027849998638, "nonpadding_tokens": 18542, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3126326330000211, "nonpadding_tokens": 4302, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3156020940000417, "nonpadding_tokens": 4915, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3188006140001107, "nonpadding_tokens": 8271, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31314378999991277, "nonpadding_tokens": 6154, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003454390000570129, "prefetch": {"data_wait_seconds": 0.025390769999788176, "consumed_batches": 2708, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3258, "tokens": 235342361, "loss_tokens": 49419, "loss": 2.3842438194317976, "lr": 0.0002987562422970247, "optimizer_lrs": [0.019917082819801646, 0.0002987562422970247], "gradient_norm": 0.42966192960739136, "seconds": 0.27967682300004526, "nonpadding_tokens_per_second": 176700.37677734919, "ranks": [{"compute_seconds": 0.27722244699998555, "nonpadding_tokens": 2201, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2739171410000836, "nonpadding_tokens": 6189, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2733433459998196, "nonpadding_tokens": 2168, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2769796039999619, "nonpadding_tokens": 3335, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2721692080001503, "nonpadding_tokens": 11319, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2744017109998822, "nonpadding_tokens": 2453, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27253541699997186, "nonpadding_tokens": 8448, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27319845399983933, "nonpadding_tokens": 13306, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035807099993689917, "prefetch": {"data_wait_seconds": 0.025399900999858005, "consumed_batches": 2709, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3259, "tokens": 235392316, "loss_tokens": 49955, "loss": 2.1490493006205584, "lr": 0.0002987556660658829, "optimizer_lrs": [0.019917044404392195, 0.0002987556660658829], "gradient_norm": 0.44444555044174194, "seconds": 0.22505470000010064, "nonpadding_tokens_per_second": 221968.25927197994, "ranks": [{"compute_seconds": 0.21874779299992042, "nonpadding_tokens": 3424, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.221354779999956, "nonpadding_tokens": 8208, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2211873349999678, "nonpadding_tokens": 10367, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21870847900004264, "nonpadding_tokens": 5380, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22211231200003567, "nonpadding_tokens": 1157, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21989155799997206, "nonpadding_tokens": 8097, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21717963700007203, "nonpadding_tokens": 10385, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22258249399988017, "nonpadding_tokens": 2937, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004279229999610834, "prefetch": {"data_wait_seconds": 0.025409164999885547, "consumed_batches": 2710, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3260, "tokens": 235473551, "loss_tokens": 81235, "loss": 2.405528712993168, "lr": 0.00029875472873616574, "optimizer_lrs": [0.019916981915744383, 0.00029875472873616574], "gradient_norm": 1.5709589719772339, "seconds": 0.2559521840000798, "nonpadding_tokens_per_second": 317383.50003676733, "ranks": [{"compute_seconds": 0.25241906700011896, "nonpadding_tokens": 7263, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25192576500012365, "nonpadding_tokens": 4264, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24874460900014128, "nonpadding_tokens": 11947, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24964852699986295, "nonpadding_tokens": 10923, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2479385070000717, "nonpadding_tokens": 9498, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25080171900003734, "nonpadding_tokens": 7948, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2533308439999473, "nonpadding_tokens": 9475, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24874243800013573, "nonpadding_tokens": 19917, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000359976000027018, "prefetch": {"data_wait_seconds": 0.025417799999743806, "consumed_batches": 2711, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3261, "tokens": 235533435, "loss_tokens": 59884, "loss": 2.3704565806392357, "lr": 0.00029875403753996114, "optimizer_lrs": [0.01991693583599741, 0.00029875403753996114], "gradient_norm": 0.44748061895370483, "seconds": 0.27542043700009344, "nonpadding_tokens_per_second": 217427.5832696456, "ranks": [{"compute_seconds": 0.2705688599999121, "nonpadding_tokens": 2231, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27309193199994297, "nonpadding_tokens": 2979, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26981795599999714, "nonpadding_tokens": 9826, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2709238469999491, "nonpadding_tokens": 14184, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26905521700018653, "nonpadding_tokens": 14346, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27184759600004327, "nonpadding_tokens": 6075, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26888196400000197, "nonpadding_tokens": 6800, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2696051969999189, "nonpadding_tokens": 3443, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004029060000902973, "prefetch": {"data_wait_seconds": 0.025428236999644582, "consumed_batches": 2712, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3262, "tokens": 235600111, "loss_tokens": 66676, "loss": 2.092106558206851, "lr": 0.0002987532677241959, "optimizer_lrs": [0.019916884514946394, 0.0002987532677241959], "gradient_norm": 0.40697234869003296, "seconds": 0.2422697160000098, "nonpadding_tokens_per_second": 275213.92727433296, "ranks": [{"compute_seconds": 0.1708133109998471, "nonpadding_tokens": 4505, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.17635835700002644, "nonpadding_tokens": 2949, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.1731229039999107, "nonpadding_tokens": 12274, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.17687720399999307, "nonpadding_tokens": 11307, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.17206579300000158, "nonpadding_tokens": 8313, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1763168869999845, "nonpadding_tokens": 11355, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.17871512100009568, "nonpadding_tokens": 3845, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.17252315700011422, "nonpadding_tokens": 12128, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.03123999499985075, "prefetch": {"data_wait_seconds": 0.025436953999545153, "consumed_batches": 2713, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3263, "tokens": 235650216, "loss_tokens": 50105, "loss": 1.9131260290889134, "lr": 0.00029875268907517247, "optimizer_lrs": [0.01991684593834483, 0.00029875268907517247], "gradient_norm": 0.460625022649765, "seconds": 0.22364685200000167, "nonpadding_tokens_per_second": 224036.2408499254, "ranks": [{"compute_seconds": 0.22086426500004563, "nonpadding_tokens": 2301, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2200048600000173, "nonpadding_tokens": 14685, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21915607300002193, "nonpadding_tokens": 2349, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2196386570001323, "nonpadding_tokens": 1744, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2178798969998752, "nonpadding_tokens": 12786, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22123477800005276, "nonpadding_tokens": 6277, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21813091300009546, "nonpadding_tokens": 5446, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21879551600000013, "nonpadding_tokens": 4517, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046417400017162436, "prefetch": {"data_wait_seconds": 0.02544747099950939, "consumed_batches": 2714, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3264, "tokens": 235709351, "loss_tokens": 59135, "loss": 2.1932421577745838, "lr": 0.00029875200596920853, "optimizer_lrs": [0.019916800397947236, 0.00029875200596920853], "gradient_norm": 0.41819143295288086, "seconds": 0.2833212479999929, "nonpadding_tokens_per_second": 208720.66750179455, "ranks": [{"compute_seconds": 0.22169158999986394, "nonpadding_tokens": 6258, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2326761289998558, "nonpadding_tokens": 14675, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2321917060000942, "nonpadding_tokens": 9144, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2300222759999997, "nonpadding_tokens": 3903, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2286768829999346, "nonpadding_tokens": 6932, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.230637913999999, "nonpadding_tokens": 3529, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22864638400005788, "nonpadding_tokens": 5787, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22927422899988414, "nonpadding_tokens": 8907, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.025885619000064253, "prefetch": {"data_wait_seconds": 0.02545605499945225, "consumed_batches": 2715, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3265, "tokens": 235788616, "loss_tokens": 79265, "loss": 2.1159269933135683, "lr": 0.00029875109003685903, "optimizer_lrs": [0.019916739335790603, 0.00029875109003685903], "gradient_norm": 0.5187870860099792, "seconds": 0.2630354210000405, "nonpadding_tokens_per_second": 301347.247069009, "ranks": [{"compute_seconds": 0.2600993019998441, "nonpadding_tokens": 8803, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26073904600002606, "nonpadding_tokens": 13170, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25798348100011026, "nonpadding_tokens": 2872, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25831011300010687, "nonpadding_tokens": 3087, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2570901120000144, "nonpadding_tokens": 10258, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2587829189999411, "nonpadding_tokens": 7047, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25668728499999816, "nonpadding_tokens": 11620, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25757701499992436, "nonpadding_tokens": 22408, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003732269999545679, "prefetch": {"data_wait_seconds": 0.025465627999395224, "consumed_batches": 2716, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3266, "tokens": 235855888, "loss_tokens": 67272, "loss": 2.5735349588238794, "lr": 0.000298750312425093, "optimizer_lrs": [0.0199166874950062, 0.000298750312425093], "gradient_norm": 0.4574010670185089, "seconds": 0.2892757959998562, "nonpadding_tokens_per_second": 232553.15837082147, "ranks": [{"compute_seconds": 0.28684087600004204, "nonpadding_tokens": 3745, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28582564299995283, "nonpadding_tokens": 4021, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28325893900000665, "nonpadding_tokens": 4246, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28619630999992296, "nonpadding_tokens": 12228, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2868649719998757, "nonpadding_tokens": 14650, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28463861800014456, "nonpadding_tokens": 10775, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2819029429999773, "nonpadding_tokens": 4043, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2825545119999333, "nonpadding_tokens": 13564, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041317399995932647, "prefetch": {"data_wait_seconds": 0.0254748759993646, "consumed_batches": 2717, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3267, "tokens": 235890665, "loss_tokens": 34777, "loss": 2.3117117185927483, "lr": 0.00029874991033569025, "optimizer_lrs": [0.019916660689046017, 0.00029874991033569025], "gradient_norm": 0.46596524119377136, "seconds": 0.25568229900000006, "nonpadding_tokens_per_second": 136016.45532763295, "ranks": [{"compute_seconds": 0.2501738620001106, "nonpadding_tokens": 3525, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.250953225999865, "nonpadding_tokens": 2217, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2484256169998389, "nonpadding_tokens": 3261, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25311104599995815, "nonpadding_tokens": 1504, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24742109099997833, "nonpadding_tokens": 2355, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2509581489998709, "nonpadding_tokens": 5490, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24722949100009828, "nonpadding_tokens": 8559, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24774166700012756, "nonpadding_tokens": 7866, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003537830000368558, "prefetch": {"data_wait_seconds": 0.025484020999286372, "consumed_batches": 2718, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3268, "tokens": 235968520, "loss_tokens": 77855, "loss": 2.0407414022863013, "lr": 0.000298749009947891, "optimizer_lrs": [0.019916600663192733, 0.000298749009947891], "gradient_norm": 0.35662156343460083, "seconds": 0.28948623800010864, "nonpadding_tokens_per_second": 268941.9729858481, "ranks": [{"compute_seconds": 0.25240140200003225, "nonpadding_tokens": 11850, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2594003210001574, "nonpadding_tokens": 14248, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2599888520001059, "nonpadding_tokens": 6732, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26011162999998305, "nonpadding_tokens": 16923, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2554775910000444, "nonpadding_tokens": 9045, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2581156749999991, "nonpadding_tokens": 7195, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25534376399991743, "nonpadding_tokens": 1868, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2615497019999111, "nonpadding_tokens": 9994, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.022346135999896433, "prefetch": {"data_wait_seconds": 0.025493010999255716, "consumed_batches": 2719, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3269, "tokens": 236019766, "loss_tokens": 51246, "loss": 2.2644935092495024, "lr": 0.0002987484171153011, "optimizer_lrs": [0.019916561141020077, 0.0002987484171153011], "gradient_norm": 0.4172450304031372, "seconds": 0.21792018899986942, "nonpadding_tokens_per_second": 235159.48767844867, "ranks": [{"compute_seconds": 0.2151281479998488, "nonpadding_tokens": 4021, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2155901650000942, "nonpadding_tokens": 4890, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2123353380000026, "nonpadding_tokens": 9261, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21302987200010648, "nonpadding_tokens": 4371, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2110215430000153, "nonpadding_tokens": 7311, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2124802980001732, "nonpadding_tokens": 8361, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21049423399995248, "nonpadding_tokens": 7186, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2118150879998666, "nonpadding_tokens": 5845, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035271000001557695, "prefetch": {"data_wait_seconds": 0.025500656999113858, "consumed_batches": 2720, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3270, "tokens": 236091231, "loss_tokens": 71465, "loss": 2.27197010774505, "lr": 0.0002987475901484904, "optimizer_lrs": [0.01991650600989936, 0.0002987475901484904], "gradient_norm": 0.46831271052360535, "seconds": 0.3484732979998171, "nonpadding_tokens_per_second": 205080.27561996304, "ranks": [{"compute_seconds": 0.34281553200003145, "nonpadding_tokens": 3513, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3419632540001203, "nonpadding_tokens": 5317, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34476543100004164, "nonpadding_tokens": 14211, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3423870120000174, "nonpadding_tokens": 6219, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.34103401299989855, "nonpadding_tokens": 2675, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34367418700003327, "nonpadding_tokens": 16936, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34595948000014687, "nonpadding_tokens": 10420, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34611952300019766, "nonpadding_tokens": 12174, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035359000003154506, "prefetch": {"data_wait_seconds": 0.025508715999151832, "consumed_batches": 2721, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3271, "tokens": 236204774, "loss_tokens": 113543, "loss": 3.751683833437552, "lr": 0.00029874627571142625, "optimizer_lrs": [0.01991641838076175, 0.00029874627571142625], "gradient_norm": 8.801138877868652, "seconds": 0.3126501520000602, "nonpadding_tokens_per_second": 363163.1050669636, "ranks": [{"compute_seconds": 0.3058973390000119, "nonpadding_tokens": 9430, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30643681200012907, "nonpadding_tokens": 16997, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30490441500001, "nonpadding_tokens": 19656, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3102236099998663, "nonpadding_tokens": 8859, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3040437260001454, "nonpadding_tokens": 11849, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3075383039999906, "nonpadding_tokens": 19916, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30389812599992183, "nonpadding_tokens": 15065, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30475976000002447, "nonpadding_tokens": 11771, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003420479999931558, "prefetch": {"data_wait_seconds": 0.025517417999211034, "consumed_batches": 2722, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3272, "tokens": 236274872, "loss_tokens": 70098, "loss": 2.154220921067648, "lr": 0.0002987454638753159, "optimizer_lrs": [0.019916364258354396, 0.0002987454638753159], "gradient_norm": 1.3381290435791016, "seconds": 0.3038619589999598, "nonpadding_tokens_per_second": 230690.2786735778, "ranks": [{"compute_seconds": 0.29713668399995186, "nonpadding_tokens": 9388, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29922393099991496, "nonpadding_tokens": 7717, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29952958199987734, "nonpadding_tokens": 12562, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29639438899994275, "nonpadding_tokens": 3963, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29544483100016805, "nonpadding_tokens": 1500, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29803779299982125, "nonpadding_tokens": 11107, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30121732199995677, "nonpadding_tokens": 19170, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.295737465000002, "nonpadding_tokens": 4691, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046060099998612714, "prefetch": {"data_wait_seconds": 0.025526776999143408, "consumed_batches": 2723, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3273, "tokens": 236332512, "loss_tokens": 57640, "loss": 2.6626312022900764, "lr": 0.0002987447961248986, "optimizer_lrs": [0.019916319741659905, 0.0002987447961248986], "gradient_norm": 0.5596494078636169, "seconds": 0.29906351199997516, "nonpadding_tokens_per_second": 192734.97998647453, "ranks": [{"compute_seconds": 0.2941608629998882, "nonpadding_tokens": 1955, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2941827929998908, "nonpadding_tokens": 2262, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29062195699998483, "nonpadding_tokens": 7022, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29162244599979203, "nonpadding_tokens": 3599, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2901777580000271, "nonpadding_tokens": 3374, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2936422729999322, "nonpadding_tokens": 10503, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2965265200000431, "nonpadding_tokens": 9370, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29039547200000015, "nonpadding_tokens": 19555, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033280699994975294, "prefetch": {"data_wait_seconds": 0.025535021999033347, "consumed_batches": 2724, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3274, "tokens": 236420298, "loss_tokens": 87786, "loss": 2.1639505445059575, "lr": 0.00029874377879802273, "optimizer_lrs": [0.019916251919868184, 0.00029874377879802273], "gradient_norm": 0.4477285146713257, "seconds": 0.30471596799998224, "nonpadding_tokens_per_second": 288091.23649209325, "ranks": [{"compute_seconds": 0.3021598719999474, "nonpadding_tokens": 13840, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29968184100016515, "nonpadding_tokens": 16746, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3009153299999525, "nonpadding_tokens": 24728, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2970169259999693, "nonpadding_tokens": 9911, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3021319209999547, "nonpadding_tokens": 11427, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2983336820000204, "nonpadding_tokens": 2387, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30166791399983595, "nonpadding_tokens": 4959, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2961098190000939, "nonpadding_tokens": 3788, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003953940001792944, "prefetch": {"data_wait_seconds": 0.025543257999018465, "consumed_batches": 2725, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3275, "tokens": 236482265, "loss_tokens": 61967, "loss": 2.1615499479561704, "lr": 0.00029874306043320523, "optimizer_lrs": [0.01991620402888035, 0.00029874306043320523], "gradient_norm": 0.4572901725769043, "seconds": 0.30417398300005516, "nonpadding_tokens_per_second": 203722.22301467764, "ranks": [{"compute_seconds": 0.2981515940000463, "nonpadding_tokens": 7740, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29729677399996035, "nonpadding_tokens": 1891, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29993811000008463, "nonpadding_tokens": 11971, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29715422000003855, "nonpadding_tokens": 5316, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2951791420000518, "nonpadding_tokens": 12825, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29809817899990776, "nonpadding_tokens": 3582, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.295520568000029, "nonpadding_tokens": 14311, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30169398599991837, "nonpadding_tokens": 4331, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039886199988359294, "prefetch": {"data_wait_seconds": 0.025553340999024954, "consumed_batches": 2726, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3276, "tokens": 236543060, "loss_tokens": 60795, "loss": 2.207962209063245, "lr": 0.0002987423554564158, "optimizer_lrs": [0.01991615703042772, 0.0002987423554564158], "gradient_norm": 0.42231303453445435, "seconds": 0.3710924160000104, "nonpadding_tokens_per_second": 163827.11523804974, "ranks": [{"compute_seconds": 0.31044945700000426, "nonpadding_tokens": 9839, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31075562599994555, "nonpadding_tokens": 7110, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3077833259999352, "nonpadding_tokens": 1677, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30828614100005325, "nonpadding_tokens": 6953, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.314218136999898, "nonpadding_tokens": 8090, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3099104929999612, "nonpadding_tokens": 7560, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31381443899999795, "nonpadding_tokens": 10025, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31467831700001625, "nonpadding_tokens": 9541, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.016844818999970812, "prefetch": {"data_wait_seconds": 0.025562580999078932, "consumed_batches": 2727, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3277, "tokens": 236608873, "loss_tokens": 65813, "loss": 2.467467958458055, "lr": 0.00029874159206932146, "optimizer_lrs": [0.019916106137954765, 0.00029874159206932146], "gradient_norm": 0.3993830382823944, "seconds": 0.23526630200012733, "nonpadding_tokens_per_second": 279738.31968491763, "ranks": [{"compute_seconds": 0.19084160900001734, "nonpadding_tokens": 2428, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19557701600001565, "nonpadding_tokens": 8127, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.19777039099994909, "nonpadding_tokens": 9151, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.19792540699995698, "nonpadding_tokens": 7203, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.19156804399995053, "nonpadding_tokens": 10185, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.19569282900010876, "nonpadding_tokens": 8984, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19140667200008465, "nonpadding_tokens": 16221, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.19265538800004833, "nonpadding_tokens": 3514, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.020551251000142656, "prefetch": {"data_wait_seconds": 0.025571026999116953, "consumed_batches": 2728, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3278, "tokens": 236676206, "loss_tokens": 67333, "loss": 2.2981290284852895, "lr": 0.0002987408108126568, "optimizer_lrs": [0.01991605405417712, 0.0002987408108126568], "gradient_norm": 0.4586985111236572, "seconds": 0.28624334400001317, "nonpadding_tokens_per_second": 235229.92380915207, "ranks": [{"compute_seconds": 0.2821886839999479, "nonpadding_tokens": 8067, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28342477500018504, "nonpadding_tokens": 7726, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28284712200002105, "nonpadding_tokens": 14406, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2809811090000949, "nonpadding_tokens": 12978, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.283898458999829, "nonpadding_tokens": 10008, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28195919699987826, "nonpadding_tokens": 4048, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28347915400013335, "nonpadding_tokens": 4127, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27977336700018895, "nonpadding_tokens": 5973, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039132400002017675, "prefetch": {"data_wait_seconds": 0.025581518999118202, "consumed_batches": 2729, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3279, "tokens": 236752416, "loss_tokens": 76210, "loss": 2.19763031426322, "lr": 0.00029873992626611563, "optimizer_lrs": [0.01991599508440771, 0.00029873992626611563], "gradient_norm": 0.4300106465816498, "seconds": 0.29102191099991614, "nonpadding_tokens_per_second": 261870.31669935657, "ranks": [{"compute_seconds": 0.284907530999817, "nonpadding_tokens": 5096, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.284658262999983, "nonpadding_tokens": 11660, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28243068000006133, "nonpadding_tokens": 4555, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2878137209997931, "nonpadding_tokens": 7305, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28153292799993324, "nonpadding_tokens": 5139, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2839291580000918, "nonpadding_tokens": 11877, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28779277200010256, "nonpadding_tokens": 13834, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28873642200005634, "nonpadding_tokens": 16744, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003391899999769521, "prefetch": {"data_wait_seconds": 0.025590021999008172, "consumed_batches": 2730, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3280, "tokens": 236852222, "loss_tokens": 99806, "loss": 2.1192397626395207, "lr": 0.0002987387673804027, "optimizer_lrs": [0.01991591782536018, 0.0002987387673804027], "gradient_norm": 0.4824698865413666, "seconds": 0.3148415829998612, "nonpadding_tokens_per_second": 317003.8692126764, "ranks": [{"compute_seconds": 0.29449289099989073, "nonpadding_tokens": 17208, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29651975000001585, "nonpadding_tokens": 14800, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29531867100013187, "nonpadding_tokens": 2274, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29972237399988444, "nonpadding_tokens": 22543, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2944968140000128, "nonpadding_tokens": 9013, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2975498890000381, "nonpadding_tokens": 7863, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29463517699991826, "nonpadding_tokens": 4183, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30082701500009534, "nonpadding_tokens": 21922, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.010264860999996017, "prefetch": {"data_wait_seconds": 0.02559875399920486, "consumed_batches": 2731, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3281, "tokens": 236914041, "loss_tokens": 61819, "loss": 2.3725439893074944, "lr": 0.00029873804931041545, "optimizer_lrs": [0.019915869954027696, 0.00029873804931041545], "gradient_norm": 0.40660223364830017, "seconds": 0.30193951500018557, "nonpadding_tokens_per_second": 204739.6810581815, "ranks": [{"compute_seconds": 0.29494166699987545, "nonpadding_tokens": 13521, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2970695149999756, "nonpadding_tokens": 10418, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2990154290000646, "nonpadding_tokens": 8234, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2947419799997988, "nonpadding_tokens": 8160, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29959291599993776, "nonpadding_tokens": 2663, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2963316479999776, "nonpadding_tokens": 1994, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2928936860000704, "nonpadding_tokens": 12484, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29410234700003457, "nonpadding_tokens": 4345, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042424100001881015, "prefetch": {"data_wait_seconds": 0.025608787999274796, "consumed_batches": 2732, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3282, "tokens": 236965820, "loss_tokens": 51779, "loss": 2.316079968471774, "lr": 0.0002987374477053711, "optimizer_lrs": [0.01991582984702474, 0.0002987374477053711], "gradient_norm": 0.4024942219257355, "seconds": 0.22967586999993728, "nonpadding_tokens_per_second": 225443.79607668033, "ranks": [{"compute_seconds": 0.2266923869999573, "nonpadding_tokens": 2874, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22601285900009316, "nonpadding_tokens": 2692, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2232816159998947, "nonpadding_tokens": 5602, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22339789600005133, "nonpadding_tokens": 13375, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22260476899987225, "nonpadding_tokens": 4018, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2246758779999709, "nonpadding_tokens": 6681, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22725160000004507, "nonpadding_tokens": 11655, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22285674800014021, "nonpadding_tokens": 4882, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000358790999825942, "prefetch": {"data_wait_seconds": 0.025617783999180688, "consumed_batches": 2733, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3283, "tokens": 237034168, "loss_tokens": 68348, "loss": 2.049311519722596, "lr": 0.00029873665337147636, "optimizer_lrs": [0.019915776891431758, 0.00029873665337147636], "gradient_norm": 0.44132333993911743, "seconds": 0.29938599399997656, "nonpadding_tokens_per_second": 228293.9127740403, "ranks": [{"compute_seconds": 0.29667973400000847, "nonpadding_tokens": 4560, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29368473999988964, "nonpadding_tokens": 11954, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2959905879999951, "nonpadding_tokens": 12580, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29680839800016656, "nonpadding_tokens": 6676, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2936381370000163, "nonpadding_tokens": 11475, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2960298830000738, "nonpadding_tokens": 2319, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29383911399986573, "nonpadding_tokens": 2725, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2938763340000605, "nonpadding_tokens": 16059, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003974280000420549, "prefetch": {"data_wait_seconds": 0.02562769799915543, "consumed_batches": 2734, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3284, "tokens": 237112775, "loss_tokens": 78607, "loss": 2.20865190123017, "lr": 0.00029873573950106624, "optimizer_lrs": [0.01991571596673775, 0.00029873573950106624], "gradient_norm": 0.3968461751937866, "seconds": 0.34106786200004535, "nonpadding_tokens_per_second": 230473.19539004102, "ranks": [{"compute_seconds": 0.3372009330000765, "nonpadding_tokens": 8168, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33626309000010224, "nonpadding_tokens": 2539, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3378504699999212, "nonpadding_tokens": 20951, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3384631129999889, "nonpadding_tokens": 5169, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3322949329999574, "nonpadding_tokens": 3996, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33470713500014426, "nonpadding_tokens": 16027, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3387046640000335, "nonpadding_tokens": 10487, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33271554799989644, "nonpadding_tokens": 11270, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003653930000382388, "prefetch": {"data_wait_seconds": 0.025635926999214576, "consumed_batches": 2735, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3285, "tokens": 237155858, "loss_tokens": 43083, "loss": 2.4062995047350464, "lr": 0.0002987352384865802, "optimizer_lrs": [0.019915682565772017, 0.0002987352384865802], "gradient_norm": 0.47425946593284607, "seconds": 0.31158260600000176, "nonpadding_tokens_per_second": 138271.51827595843, "ranks": [{"compute_seconds": 0.3061155959999269, "nonpadding_tokens": 1660, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30767563400013387, "nonpadding_tokens": 5400, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30475804800016704, "nonpadding_tokens": 3123, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30494630499993036, "nonpadding_tokens": 3593, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30900067499987927, "nonpadding_tokens": 4018, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30673011099997893, "nonpadding_tokens": 18667, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30346992899990255, "nonpadding_tokens": 4931, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30927202399993803, "nonpadding_tokens": 1691, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003448929999194661, "prefetch": {"data_wait_seconds": 0.02564445999924203, "consumed_batches": 2736, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3286, "tokens": 237193244, "loss_tokens": 37386, "loss": 2.0763510655726742, "lr": 0.0002987348036427524, "optimizer_lrs": [0.01991565357618349, 0.0002987348036427524], "gradient_norm": 0.4701739549636841, "seconds": 0.24076560799994695, "nonpadding_tokens_per_second": 155279.65273183136, "ranks": [{"compute_seconds": 0.22984269800008406, "nonpadding_tokens": 5727, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2341496950000419, "nonpadding_tokens": 6054, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23469914600013908, "nonpadding_tokens": 2206, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23193250999997872, "nonpadding_tokens": 2814, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23105367800008025, "nonpadding_tokens": 1454, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2328286530000696, "nonpadding_tokens": 3187, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23599963500009835, "nonpadding_tokens": 5474, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23120717000006152, "nonpadding_tokens": 10470, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003353780000452389, "prefetch": {"data_wait_seconds": 0.02565353299924311, "consumed_batches": 2737, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3287, "tokens": 237288713, "loss_tokens": 95469, "loss": 2.376375610145702, "lr": 0.000298733692886753, "optimizer_lrs": [0.019915579525783534, 0.000298733692886753], "gradient_norm": 0.44428107142448425, "seconds": 0.3725715850000597, "nonpadding_tokens_per_second": 256243.3740082049, "ranks": [{"compute_seconds": 0.3669428370001242, "nonpadding_tokens": 21433, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36631687800013424, "nonpadding_tokens": 14363, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36536245499996767, "nonpadding_tokens": 5088, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3655101839999588, "nonpadding_tokens": 9741, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3701397279999128, "nonpadding_tokens": 11873, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3671341780000148, "nonpadding_tokens": 9029, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36951218800004426, "nonpadding_tokens": 9722, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3649492619999819, "nonpadding_tokens": 14220, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003455699998085038, "prefetch": {"data_wait_seconds": 0.025661316999233463, "consumed_batches": 2738, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3288, "tokens": 237380303, "loss_tokens": 91590, "loss": 2.2766390981548206, "lr": 0.00029873262680596023, "optimizer_lrs": [0.019915508453730683, 0.00029873262680596023], "gradient_norm": 0.5373421311378479, "seconds": 0.2614142729999003, "nonpadding_tokens_per_second": 350363.4248770913, "ranks": [{"compute_seconds": 0.2579905879999842, "nonpadding_tokens": 5610, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.256838585999958, "nonpadding_tokens": 19065, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2577254999998786, "nonpadding_tokens": 11252, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2544726969999829, "nonpadding_tokens": 9689, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25854101200002333, "nonpadding_tokens": 13908, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2554644599999847, "nonpadding_tokens": 10887, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2581574159999036, "nonpadding_tokens": 11622, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25905615000010584, "nonpadding_tokens": 9557, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003381329997864668, "prefetch": {"data_wait_seconds": 0.025669140999028173, "consumed_batches": 2739, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3289, "tokens": 237463052, "loss_tokens": 82749, "loss": 2.2855385714630994, "lr": 0.00029873166324793064, "optimizer_lrs": [0.01991544421652871, 0.00029873166324793064], "gradient_norm": 0.44336840510368347, "seconds": 0.3341996999999992, "nonpadding_tokens_per_second": 247603.4538630651, "ranks": [{"compute_seconds": 0.33069132199989326, "nonpadding_tokens": 4226, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32908530000008795, "nonpadding_tokens": 13961, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32650270099998124, "nonpadding_tokens": 9550, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3309500919999664, "nonpadding_tokens": 6468, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32559515699995245, "nonpadding_tokens": 6576, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3283214069999758, "nonpadding_tokens": 12946, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.325985725999999, "nonpadding_tokens": 8715, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3317884650000451, "nonpadding_tokens": 20307, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004015849999632337, "prefetch": {"data_wait_seconds": 0.025677924998902313, "consumed_batches": 2740, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3290, "tokens": 237514946, "loss_tokens": 51894, "loss": 1.9751402799456586, "lr": 0.00029873105879028394, "optimizer_lrs": [0.019915403919352263, 0.00029873105879028394], "gradient_norm": 0.48715031147003174, "seconds": 0.24130798100009088, "nonpadding_tokens_per_second": 215052.9782932478, "ranks": [{"compute_seconds": 0.2385499219999474, "nonpadding_tokens": 3487, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23473377799996342, "nonpadding_tokens": 4737, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23252348999994865, "nonpadding_tokens": 4661, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23319623599991246, "nonpadding_tokens": 10269, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2313674130000436, "nonpadding_tokens": 2379, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23563916300008714, "nonpadding_tokens": 15687, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23116201900006672, "nonpadding_tokens": 4215, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2384744349999437, "nonpadding_tokens": 6459, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003860260001147253, "prefetch": {"data_wait_seconds": 0.025687240999062055, "consumed_batches": 2741, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3291, "tokens": 237588365, "loss_tokens": 73419, "loss": 2.198756878328498, "lr": 0.00029873020336613717, "optimizer_lrs": [0.019915346891075814, 0.00029873020336613717], "gradient_norm": 0.4670540988445282, "seconds": 0.369294704999902, "nonpadding_tokens_per_second": 198808.69940991836, "ranks": [{"compute_seconds": 0.36343039500002305, "nonpadding_tokens": 21076, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3636936959999275, "nonpadding_tokens": 10397, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3652376819998153, "nonpadding_tokens": 8441, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3656257220000043, "nonpadding_tokens": 9614, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3662447609999617, "nonpadding_tokens": 4648, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3627463890002218, "nonpadding_tokens": 3371, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36616454699992573, "nonpadding_tokens": 4453, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.36677392299998246, "nonpadding_tokens": 11419, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038507300018864044, "prefetch": {"data_wait_seconds": 0.02569702799901563, "consumed_batches": 2742, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3292, "tokens": 237641266, "loss_tokens": 52901, "loss": 2.4489047052513184, "lr": 0.00029872958682480524, "optimizer_lrs": [0.01991530578832035, 0.00029872958682480524], "gradient_norm": 0.44188547134399414, "seconds": 0.2649735469999541, "nonpadding_tokens_per_second": 199646.3443198319, "ranks": [{"compute_seconds": 0.2625683020000906, "nonpadding_tokens": 1079, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2590842999998131, "nonpadding_tokens": 6304, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25913178099995093, "nonpadding_tokens": 2687, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25930449299994507, "nonpadding_tokens": 11213, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25799707600003785, "nonpadding_tokens": 11089, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2599693729998762, "nonpadding_tokens": 7942, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25763993800001117, "nonpadding_tokens": 2641, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.258723793000172, "nonpadding_tokens": 9946, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003865400001359376, "prefetch": {"data_wait_seconds": 0.025705683999106554, "consumed_batches": 2743, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3293, "tokens": 237717707, "loss_tokens": 76441, "loss": 2.0907529990450153, "lr": 0.0002987286956705044, "optimizer_lrs": [0.019915246378033626, 0.0002987286956705044], "gradient_norm": 0.5941513776779175, "seconds": 0.3091641180001261, "nonpadding_tokens_per_second": 247250.55577105755, "ranks": [{"compute_seconds": 0.30662345200016716, "nonpadding_tokens": 19303, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3059676269999727, "nonpadding_tokens": 11634, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3026950250000482, "nonpadding_tokens": 6813, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3059322029998839, "nonpadding_tokens": 3046, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30156804700004614, "nonpadding_tokens": 3100, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30441746399992553, "nonpadding_tokens": 7092, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30158503100005873, "nonpadding_tokens": 16589, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30267464199982896, "nonpadding_tokens": 8864, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00046904200007702457, "prefetch": {"data_wait_seconds": 0.025715846999176506, "consumed_batches": 2744, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3294, "tokens": 237782297, "loss_tokens": 64590, "loss": 2.1929214564948136, "lr": 0.00029872794243356526, "optimizer_lrs": [0.019915196162237685, 0.00029872794243356526], "gradient_norm": 0.4041769802570343, "seconds": 0.3093480439999894, "nonpadding_tokens_per_second": 208793.94989807086, "ranks": [{"compute_seconds": 0.3056839349999336, "nonpadding_tokens": 9284, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3048935360000087, "nonpadding_tokens": 3465, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30156944199984537, "nonpadding_tokens": 3172, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30257199299990134, "nonpadding_tokens": 9685, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3070048360000328, "nonpadding_tokens": 12756, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3046695479999926, "nonpadding_tokens": 7125, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30048423800008095, "nonpadding_tokens": 13111, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30150521500013383, "nonpadding_tokens": 5992, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004333419999511534, "prefetch": {"data_wait_seconds": 0.025725616999309864, "consumed_batches": 2745, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3295, "tokens": 237846982, "loss_tokens": 64685, "loss": 2.062794938934838, "lr": 0.00029872718786624224, "optimizer_lrs": [0.019915145857749485, 0.00029872718786624224], "gradient_norm": 0.41118940711021423, "seconds": 0.28364608399988356, "nonpadding_tokens_per_second": 228048.27441237142, "ranks": [{"compute_seconds": 0.27844887200012636, "nonpadding_tokens": 6306, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2783777799998006, "nonpadding_tokens": 6714, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27542749500003083, "nonpadding_tokens": 4554, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27582669599996734, "nonpadding_tokens": 15633, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2813265560000673, "nonpadding_tokens": 9904, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2772635189999164, "nonpadding_tokens": 11477, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28083506999996644, "nonpadding_tokens": 4309, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2752386980000665, "nonpadding_tokens": 5788, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003440010000304028, "prefetch": {"data_wait_seconds": 0.02573475399931624, "consumed_batches": 2746, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3296, "tokens": 237909002, "loss_tokens": 62020, "loss": 2.4268144348597227, "lr": 0.00029872646417773523, "optimizer_lrs": [0.019915097611849016, 0.00029872646417773523], "gradient_norm": 0.48742929100990295, "seconds": 0.28018766000013784, "nonpadding_tokens_per_second": 221351.6469639294, "ranks": [{"compute_seconds": 0.2744400150002093, "nonpadding_tokens": 9742, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27359261000015067, "nonpadding_tokens": 7059, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2708308109999962, "nonpadding_tokens": 10965, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27100984300000164, "nonpadding_tokens": 9007, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27756315000010545, "nonpadding_tokens": 8749, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27255907600010687, "nonpadding_tokens": 10522, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26987939000014194, "nonpadding_tokens": 3425, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2703802150001593, "nonpadding_tokens": 2551, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003326080000078946, "prefetch": {"data_wait_seconds": 0.025743156999354255, "consumed_batches": 2747, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3297, "tokens": 237952738, "loss_tokens": 43736, "loss": 2.372284845436254, "lr": 0.0002987259537154052, "optimizer_lrs": [0.019915063581027015, 0.0002987259537154052], "gradient_norm": 0.5087873339653015, "seconds": 0.1969407770000089, "nonpadding_tokens_per_second": 222076.91401561812, "ranks": [{"compute_seconds": 0.19159044100001665, "nonpadding_tokens": 2886, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.1933622210001431, "nonpadding_tokens": 14197, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.19325685199987674, "nonpadding_tokens": 7074, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.19085270699997636, "nonpadding_tokens": 3474, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.1941194529999848, "nonpadding_tokens": 3590, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.1919284289999723, "nonpadding_tokens": 3956, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.18894710199992915, "nonpadding_tokens": 3326, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.19456423999986328, "nonpadding_tokens": 5233, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003664549999484734, "prefetch": {"data_wait_seconds": 0.025752671999228482, "consumed_batches": 2748, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3298, "tokens": 237999691, "loss_tokens": 46953, "loss": 2.3805788834046813, "lr": 0.000298725405592732, "optimizer_lrs": [0.01991502703951547, 0.000298725405592732], "gradient_norm": 0.4080844521522522, "seconds": 0.24097718400003032, "nonpadding_tokens_per_second": 194844.1724673573, "ranks": [{"compute_seconds": 0.23346254400007638, "nonpadding_tokens": 4981, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23424406799995268, "nonpadding_tokens": 3987, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23097823399984918, "nonpadding_tokens": 4768, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23179434200005744, "nonpadding_tokens": 6191, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2300839580000229, "nonpadding_tokens": 8772, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23533059800001865, "nonpadding_tokens": 6462, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23780188099999577, "nonpadding_tokens": 2081, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23840313599998808, "nonpadding_tokens": 9711, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003343769999446522, "prefetch": {"data_wait_seconds": 0.02576011899937214, "consumed_batches": 2749, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3299, "tokens": 238058520, "loss_tokens": 58829, "loss": 2.1486580810484623, "lr": 0.00029872471866572876, "optimizer_lrs": [0.01991498124438192, 0.00029872471866572876], "gradient_norm": 0.38859236240386963, "seconds": 0.28607708799995635, "nonpadding_tokens_per_second": 205640.37620520304, "ranks": [{"compute_seconds": 0.28154640399998243, "nonpadding_tokens": 5198, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.281537169999865, "nonpadding_tokens": 12008, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27845532700007425, "nonpadding_tokens": 9732, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28274159900001905, "nonpadding_tokens": 10317, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2831781410000076, "nonpadding_tokens": 3681, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.280299745000093, "nonpadding_tokens": 4782, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2832362540000304, "nonpadding_tokens": 4793, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2835552510000525, "nonpadding_tokens": 8318, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038320899989230384, "prefetch": {"data_wait_seconds": 0.025770505999346938, "consumed_batches": 2750, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3300, "tokens": 238113971, "loss_tokens": 55451, "loss": 2.1483615602063084, "lr": 0.0002987240710139099, "optimizer_lrs": [0.019914938067593996, 0.0002987240710139099], "gradient_norm": 0.6972705721855164, "seconds": 0.32082771199998206, "nonpadding_tokens_per_second": 172837.31400360796, "ranks": [{"compute_seconds": 0.3039008399998693, "nonpadding_tokens": 15274, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31673692300000766, "nonpadding_tokens": 3273, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3174841519999063, "nonpadding_tokens": 7154, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3149077820000912, "nonpadding_tokens": 6834, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3133125229999223, "nonpadding_tokens": 8861, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3157761520001259, "nonpadding_tokens": 7190, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3183608919998733, "nonpadding_tokens": 1988, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3139994850000676, "nonpadding_tokens": 4877, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003552279999894381, "prefetch": {"data_wait_seconds": 0.025779225999443156, "consumed_batches": 2751, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3301, "tokens": 238147885, "loss_tokens": 33914, "loss": 2.1627038243793124, "lr": 0.0002987236748274475, "optimizer_lrs": [0.019914911655163167, 0.0002987236748274475], "gradient_norm": 0.4617351293563843, "seconds": 0.2526306199999908, "nonpadding_tokens_per_second": 134243.42623234363, "ranks": [{"compute_seconds": 0.2502718860000641, "nonpadding_tokens": 2195, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24660790600000837, "nonpadding_tokens": 2531, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24823453199996948, "nonpadding_tokens": 2028, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24891079400003946, "nonpadding_tokens": 6796, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2432394639999984, "nonpadding_tokens": 2528, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2455214930000693, "nonpadding_tokens": 9207, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24329874599993673, "nonpadding_tokens": 6272, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24993931699987115, "nonpadding_tokens": 2357, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039385500008393137, "prefetch": {"data_wait_seconds": 0.025789142999386172, "consumed_batches": 2752, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3302, "tokens": 238204379, "loss_tokens": 56494, "loss": 2.282517695463235, "lr": 0.0002987230147234488, "optimizer_lrs": [0.01991486764822992, 0.0002987230147234488], "gradient_norm": 0.619118332862854, "seconds": 0.35995759700017516, "nonpadding_tokens_per_second": 156946.26386777582, "ranks": [{"compute_seconds": 0.3557105469999442, "nonpadding_tokens": 14953, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35321493299989015, "nonpadding_tokens": 1958, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3555441850000989, "nonpadding_tokens": 5685, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3520233130000179, "nonpadding_tokens": 9146, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3509230849999767, "nonpadding_tokens": 8987, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3535233469999639, "nonpadding_tokens": 9076, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35060234499997023, "nonpadding_tokens": 1361, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.35744046600007096, "nonpadding_tokens": 5328, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038961099994594406, "prefetch": {"data_wait_seconds": 0.025799525999403272, "consumed_batches": 2753, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3303, "tokens": 238285513, "loss_tokens": 81134, "loss": 2.3360956103483126, "lr": 0.0002987220664163166, "optimizer_lrs": [0.01991480442775444, 0.0002987220664163166], "gradient_norm": 0.38989129662513733, "seconds": 0.2661584170000424, "nonpadding_tokens_per_second": 304833.49320486485, "ranks": [{"compute_seconds": 0.2617665029999898, "nonpadding_tokens": 9358, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2631538409998484, "nonpadding_tokens": 15250, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2625550810000732, "nonpadding_tokens": 10104, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26334388500004025, "nonpadding_tokens": 9731, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.263749684000004, "nonpadding_tokens": 1321, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2618901419998565, "nonpadding_tokens": 15493, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26327387400010593, "nonpadding_tokens": 10447, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25942433000000165, "nonpadding_tokens": 9430, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035337799999979325, "prefetch": {"data_wait_seconds": 0.02580859799941493, "consumed_batches": 2754, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3304, "tokens": 238335764, "loss_tokens": 50251, "loss": 2.2955730980478, "lr": 0.00029872147889894096, "optimizer_lrs": [0.0199147652599294, 0.00029872147889894096], "gradient_norm": 0.47048258781433105, "seconds": 0.33342591499990704, "nonpadding_tokens_per_second": 150711.1407342588, "ranks": [{"compute_seconds": 0.3305656100001215, "nonpadding_tokens": 8521, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32950393100009023, "nonpadding_tokens": 8523, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32645431700007066, "nonpadding_tokens": 4522, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32663534299990715, "nonpadding_tokens": 10049, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3256440619998102, "nonpadding_tokens": 8371, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3278691279999748, "nonpadding_tokens": 2586, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3256709510001201, "nonpadding_tokens": 3334, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33113386400009404, "nonpadding_tokens": 4345, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003615410000747943, "prefetch": {"data_wait_seconds": 0.025816417999479313, "consumed_batches": 2755, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3305, "tokens": 238397787, "loss_tokens": 62023, "loss": 2.3764029573706527, "lr": 0.00029872075356211135, "optimizer_lrs": [0.019914716904140758, 0.00029872075356211135], "gradient_norm": 0.4513143301010132, "seconds": 0.3070163830000183, "nonpadding_tokens_per_second": 202018.53527795713, "ranks": [{"compute_seconds": 0.301843738000116, "nonpadding_tokens": 5500, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3020571699998982, "nonpadding_tokens": 10325, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2985154909999892, "nonpadding_tokens": 7737, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2991908770000009, "nonpadding_tokens": 12319, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2977393450000818, "nonpadding_tokens": 2901, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3007878559999426, "nonpadding_tokens": 6800, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29821391900009075, "nonpadding_tokens": 13118, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3044735869998476, "nonpadding_tokens": 3323, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038541300000360934, "prefetch": {"data_wait_seconds": 0.025826136999469895, "consumed_batches": 2756, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3306, "tokens": 238468386, "loss_tokens": 70599, "loss": 2.2117823906854204, "lr": 0.0002987199276828964, "optimizer_lrs": [0.019914661845526427, 0.0002987199276828964], "gradient_norm": 0.4285060167312622, "seconds": 0.30982460800009903, "nonpadding_tokens_per_second": 227867.6327736286, "ranks": [{"compute_seconds": 0.30508731899999475, "nonpadding_tokens": 15913, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30372788499994385, "nonpadding_tokens": 8748, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30404548599994996, "nonpadding_tokens": 5025, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30420509200007473, "nonpadding_tokens": 3373, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3030161340000177, "nonpadding_tokens": 19665, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3060427240000081, "nonpadding_tokens": 1935, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30239151600017067, "nonpadding_tokens": 8137, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3074324849999357, "nonpadding_tokens": 7803, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003959959999519924, "prefetch": {"data_wait_seconds": 0.025834398999407426, "consumed_batches": 2757, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3307, "tokens": 238521526, "loss_tokens": 53140, "loss": 2.160529968009033, "lr": 0.0002987193058671002, "optimizer_lrs": [0.019914620391140017, 0.0002987193058671002], "gradient_norm": 0.4364077150821686, "seconds": 0.24891020900008698, "nonpadding_tokens_per_second": 213490.64071526864, "ranks": [{"compute_seconds": 0.2430252809999729, "nonpadding_tokens": 2496, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24523047700017742, "nonpadding_tokens": 6722, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24223858400000609, "nonpadding_tokens": 6829, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24281368500010103, "nonpadding_tokens": 3798, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24584184499985895, "nonpadding_tokens": 4911, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24405858699992677, "nonpadding_tokens": 13022, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24525665099986327, "nonpadding_tokens": 3920, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24654167199992116, "nonpadding_tokens": 11442, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033881100011967646, "prefetch": {"data_wait_seconds": 0.025843211999429627, "consumed_batches": 2758, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3308, "tokens": 238603048, "loss_tokens": 81522, "loss": 2.3677153483108855, "lr": 0.00029871835164825783, "optimizer_lrs": [0.01991455677655052, 0.00029871835164825783], "gradient_norm": 0.453110933303833, "seconds": 0.2723973259999184, "nonpadding_tokens_per_second": 299276.0655808509, "ranks": [{"compute_seconds": 0.26906584399989697, "nonpadding_tokens": 3492, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26908045300001504, "nonpadding_tokens": 4096, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2659506560000864, "nonpadding_tokens": 12019, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26670886700003393, "nonpadding_tokens": 22221, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2696508479998556, "nonpadding_tokens": 20775, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26755501699994966, "nonpadding_tokens": 4272, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2692866999998387, "nonpadding_tokens": 5734, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2698768350001046, "nonpadding_tokens": 8913, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033256599999731407, "prefetch": {"data_wait_seconds": 0.0258509499994517, "consumed_batches": 2759, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3309, "tokens": 238680409, "loss_tokens": 77361, "loss": 2.2591722977340005, "lr": 0.0002987174458070755, "optimizer_lrs": [0.019914496387138367, 0.0002987174458070755], "gradient_norm": 0.4011194705963135, "seconds": 0.3548622160001287, "nonpadding_tokens_per_second": 218002.92201289738, "ranks": [{"compute_seconds": 0.35239930500006267, "nonpadding_tokens": 9375, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3507886919999237, "nonpadding_tokens": 11221, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3479221659999894, "nonpadding_tokens": 6342, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3482506420000391, "nonpadding_tokens": 9293, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3524026459999732, "nonpadding_tokens": 18013, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.34990946800007805, "nonpadding_tokens": 5930, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.35177977300008934, "nonpadding_tokens": 9672, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3470780499999364, "nonpadding_tokens": 7515, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042363399984424177, "prefetch": {"data_wait_seconds": 0.025861042999352435, "consumed_batches": 2760, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3310, "tokens": 238730537, "loss_tokens": 50128, "loss": 3.0086145093560486, "lr": 0.00029871685867452426, "optimizer_lrs": [0.019914457244968285, 0.00029871685867452426], "gradient_norm": 0.4431932866573334, "seconds": 0.3743703180000466, "nonpadding_tokens_per_second": 133899.50428707266, "ranks": [{"compute_seconds": 0.3682345039999291, "nonpadding_tokens": 17583, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36957643899995674, "nonpadding_tokens": 5012, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3663661220000449, "nonpadding_tokens": 3066, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3665189470000314, "nonpadding_tokens": 7488, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3711133520000658, "nonpadding_tokens": 4170, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3683263790001092, "nonpadding_tokens": 2660, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.36504496799989283, "nonpadding_tokens": 6267, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3717747559999225, "nonpadding_tokens": 3882, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004023420001431077, "prefetch": {"data_wait_seconds": 0.02587081699944349, "consumed_batches": 2761, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3311, "tokens": 238786259, "loss_tokens": 55722, "loss": 2.3160970980941102, "lr": 0.0002987162058643829, "optimizer_lrs": [0.019914413724292195, 0.0002987162058643829], "gradient_norm": 0.45260894298553467, "seconds": 0.32178772599991134, "nonpadding_tokens_per_second": 173163.84528605468, "ranks": [{"compute_seconds": 0.31560074100002566, "nonpadding_tokens": 4045, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.315286942000057, "nonpadding_tokens": 9120, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31415950399991743, "nonpadding_tokens": 4042, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3182098439999663, "nonpadding_tokens": 3464, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3133716969998659, "nonpadding_tokens": 7113, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3166932359999919, "nonpadding_tokens": 16765, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.318731661999891, "nonpadding_tokens": 8013, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.319189630999972, "nonpadding_tokens": 3160, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003974999999627471, "prefetch": {"data_wait_seconds": 0.02587955199938108, "consumed_batches": 2762, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3312, "tokens": 238856268, "loss_tokens": 70009, "loss": 1.9721821480095416, "lr": 0.00029871538544093423, "optimizer_lrs": [0.019914359029395617, 0.00029871538544093423], "gradient_norm": 0.7070420980453491, "seconds": 0.2774980960000448, "nonpadding_tokens_per_second": 252286.41568765463, "ranks": [{"compute_seconds": 0.25191090400016947, "nonpadding_tokens": 4262, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2702190840000185, "nonpadding_tokens": 1852, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27328522800007704, "nonpadding_tokens": 13786, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2689020640000308, "nonpadding_tokens": 11642, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2679342659998838, "nonpadding_tokens": 4497, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27148917199997413, "nonpadding_tokens": 5926, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2681020630000148, "nonpadding_tokens": 25375, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2746441509998476, "nonpadding_tokens": 2669, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004128739999487152, "prefetch": {"data_wait_seconds": 0.025888491999424446, "consumed_batches": 2763, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3313, "tokens": 238944158, "loss_tokens": 87890, "loss": 2.178891405449994, "lr": 0.0002987143551038752, "optimizer_lrs": [0.019914290340258347, 0.0002987143551038752], "gradient_norm": 0.4300253093242645, "seconds": 0.28142640500004745, "nonpadding_tokens_per_second": 312301.8964762215, "ranks": [{"compute_seconds": 0.2767046769999979, "nonpadding_tokens": 5840, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2757264340000347, "nonpadding_tokens": 20255, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27609784700007367, "nonpadding_tokens": 17162, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27332919900004526, "nonpadding_tokens": 4938, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27762266099989574, "nonpadding_tokens": 17608, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2741983129999426, "nonpadding_tokens": 8248, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27680906499995217, "nonpadding_tokens": 7396, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2787717910000538, "nonpadding_tokens": 6443, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004388740001104452, "prefetch": {"data_wait_seconds": 0.02589768499952072, "consumed_batches": 2764, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3314, "tokens": 239019154, "loss_tokens": 74996, "loss": 2.333975312016641, "lr": 0.0002987134755985372, "optimizer_lrs": [0.019914231706569148, 0.0002987134755985372], "gradient_norm": 0.40367788076400757, "seconds": 0.24840807900000073, "nonpadding_tokens_per_second": 301906.4448382928, "ranks": [{"compute_seconds": 0.24518399699991278, "nonpadding_tokens": 7013, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.244566346000056, "nonpadding_tokens": 17196, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24419216199999028, "nonpadding_tokens": 15915, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2426770580000266, "nonpadding_tokens": 15572, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24103083299996797, "nonpadding_tokens": 3153, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2436675760000071, "nonpadding_tokens": 3620, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2455526960000043, "nonpadding_tokens": 10212, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24602956900002937, "nonpadding_tokens": 2315, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037388999999166117, "prefetch": {"data_wait_seconds": 0.02590737599958004, "consumed_batches": 2765, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3315, "tokens": 239097027, "loss_tokens": 77873, "loss": 2.2057201709835246, "lr": 0.0002987125620368426, "optimizer_lrs": [0.019914170802456176, 0.0002987125620368426], "gradient_norm": 0.39903581142425537, "seconds": 0.28007172699994953, "nonpadding_tokens_per_second": 278046.63053337776, "ranks": [{"compute_seconds": 0.2764215529998637, "nonpadding_tokens": 10152, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27334728399978303, "nonpadding_tokens": 8006, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27582406499982426, "nonpadding_tokens": 6256, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2732579240000632, "nonpadding_tokens": 17347, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2716202509998311, "nonpadding_tokens": 12047, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2742978960000073, "nonpadding_tokens": 13026, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27662523499998315, "nonpadding_tokens": 8979, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27758651599992845, "nonpadding_tokens": 2060, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037925400010863086, "prefetch": {"data_wait_seconds": 0.025917480999581244, "consumed_batches": 2766, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3316, "tokens": 239173084, "loss_tokens": 76057, "loss": 2.493445499756761, "lr": 0.00029871166946793476, "optimizer_lrs": [0.01991411129786232, 0.00029871166946793476], "gradient_norm": 0.3967353105545044, "seconds": 0.38042914000016026, "nonpadding_tokens_per_second": 199924.22241883984, "ranks": [{"compute_seconds": 0.3777768790000664, "nonpadding_tokens": 17724, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3757461550001153, "nonpadding_tokens": 9363, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.37604775199997675, "nonpadding_tokens": 10796, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3734089639999638, "nonpadding_tokens": 6333, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.37745818100006545, "nonpadding_tokens": 10836, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.37466249099998095, "nonpadding_tokens": 6943, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3717220659998475, "nonpadding_tokens": 7526, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3721447140001146, "nonpadding_tokens": 6536, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004235480000716052, "prefetch": {"data_wait_seconds": 0.02592986099962502, "consumed_batches": 2767, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3317, "tokens": 239221149, "loss_tokens": 48065, "loss": 1.999894348798502, "lr": 0.00029871110524114344, "optimizer_lrs": [0.019914073682742898, 0.00029871110524114344], "gradient_norm": 0.5695237517356873, "seconds": 0.28090933299995413, "nonpadding_tokens_per_second": 171105.03053313593, "ranks": [{"compute_seconds": 0.27680560800013154, "nonpadding_tokens": 13116, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27469996900003935, "nonpadding_tokens": 6290, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2775272600001699, "nonpadding_tokens": 3466, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.273133874999985, "nonpadding_tokens": 1495, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27223575800007893, "nonpadding_tokens": 4772, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27504905599994345, "nonpadding_tokens": 9813, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2783891050000875, "nonpadding_tokens": 2135, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2726836980000371, "nonpadding_tokens": 6978, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045136699986869644, "prefetch": {"data_wait_seconds": 0.02593986499955747, "consumed_batches": 2768, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3318, "tokens": 239280153, "loss_tokens": 59004, "loss": 2.1210509827723545, "lr": 0.0002987104124352243, "optimizer_lrs": [0.01991402749568162, 0.0002987104124352243], "gradient_norm": 0.4092579483985901, "seconds": 0.2830513859998973, "nonpadding_tokens_per_second": 208456.84889181715, "ranks": [{"compute_seconds": 0.2791249179999795, "nonpadding_tokens": 10731, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27755636899996716, "nonpadding_tokens": 8392, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27925703100004284, "nonpadding_tokens": 6077, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27583525699992606, "nonpadding_tokens": 4996, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2743737799999053, "nonpadding_tokens": 13420, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2774112829999922, "nonpadding_tokens": 1747, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28056918299989775, "nonpadding_tokens": 4047, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2747287080001115, "nonpadding_tokens": 9594, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003510750000259577, "prefetch": {"data_wait_seconds": 0.025948667999458053, "consumed_batches": 2769, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3319, "tokens": 239331777, "loss_tokens": 51624, "loss": 2.3633994423620797, "lr": 0.00029870980613093536, "optimizer_lrs": [0.01991398707539569, 0.00029870980613093536], "gradient_norm": 0.4082544445991516, "seconds": 0.29185016600013114, "nonpadding_tokens_per_second": 176885.28571875748, "ranks": [{"compute_seconds": 0.28775017799989655, "nonpadding_tokens": 14043, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2857430999999906, "nonpadding_tokens": 4097, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2829982220000602, "nonpadding_tokens": 3884, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2839784080001664, "nonpadding_tokens": 9512, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2892881599998418, "nonpadding_tokens": 3932, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28500134299997626, "nonpadding_tokens": 4244, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2825629469998603, "nonpadding_tokens": 5414, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28910724299998947, "nonpadding_tokens": 6498, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039243400010491314, "prefetch": {"data_wait_seconds": 0.025958319999247692, "consumed_batches": 2770, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3320, "tokens": 239395892, "loss_tokens": 64115, "loss": 2.31585237463932, "lr": 0.00029870905292716144, "optimizer_lrs": [0.019913936861810765, 0.00029870905292716144], "gradient_norm": 1.0501667261123657, "seconds": 0.32842754100011007, "nonpadding_tokens_per_second": 195218.09835058416, "ranks": [{"compute_seconds": 0.32349234999992404, "nonpadding_tokens": 5839, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3234956480000619, "nonpadding_tokens": 13172, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3205823069999951, "nonpadding_tokens": 5867, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3256810509999468, "nonpadding_tokens": 9518, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3259198779999224, "nonpadding_tokens": 2338, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3232515299998795, "nonpadding_tokens": 5592, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3258088099998986, "nonpadding_tokens": 9968, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3209588879999501, "nonpadding_tokens": 11821, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034008700004051207, "prefetch": {"data_wait_seconds": 0.025966879999216985, "consumed_batches": 2771, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3321, "tokens": 239449350, "loss_tokens": 53458, "loss": 2.297342949137641, "lr": 0.0002987084247514029, "optimizer_lrs": [0.019913894983426862, 0.0002987084247514029], "gradient_norm": 0.5267862677574158, "seconds": 0.3052646820001428, "nonpadding_tokens_per_second": 175120.16014998747, "ranks": [{"compute_seconds": 0.3026516620000166, "nonpadding_tokens": 8574, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29888747299992247, "nonpadding_tokens": 3139, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30126788799998394, "nonpadding_tokens": 5769, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30221556500009683, "nonpadding_tokens": 10048, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29600090500002807, "nonpadding_tokens": 3978, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29873057500003597, "nonpadding_tokens": 1893, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29584208500000386, "nonpadding_tokens": 10191, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29627572000003966, "nonpadding_tokens": 9866, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004072970000379428, "prefetch": {"data_wait_seconds": 0.025976686999186427, "consumed_batches": 2772, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3322, "tokens": 239502939, "loss_tokens": 53589, "loss": 2.42057954990763, "lr": 0.00029870779488366296, "optimizer_lrs": [0.0199138529922442, 0.00029870779488366296], "gradient_norm": 0.4444352388381958, "seconds": 0.32182061999992584, "nonpadding_tokens_per_second": 166518.2299381946, "ranks": [{"compute_seconds": 0.3179211359999954, "nonpadding_tokens": 7518, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3163596089998464, "nonpadding_tokens": 7705, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31796768000003794, "nonpadding_tokens": 3215, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3136956509999891, "nonpadding_tokens": 12490, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3187258809998639, "nonpadding_tokens": 4826, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3152328840001246, "nonpadding_tokens": 5595, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31859052699996937, "nonpadding_tokens": 5840, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3193607659998179, "nonpadding_tokens": 6400, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004355119999672752, "prefetch": {"data_wait_seconds": 0.025986842999330406, "consumed_batches": 2773, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3323, "tokens": 239558094, "loss_tokens": 55155, "loss": 2.4321905312301695, "lr": 0.00029870714645009397, "optimizer_lrs": [0.019913809763339597, 0.00029870714645009397], "gradient_norm": 0.43421047925949097, "seconds": 0.3391926379999859, "nonpadding_tokens_per_second": 162606.7131799078, "ranks": [{"compute_seconds": 0.3324061939999865, "nonpadding_tokens": 15248, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33293511700003364, "nonpadding_tokens": 9455, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3311735940001199, "nonpadding_tokens": 11345, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3360716939998838, "nonpadding_tokens": 3936, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3368455949998861, "nonpadding_tokens": 2789, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.33340701600013745, "nonpadding_tokens": 5099, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3301858039999388, "nonpadding_tokens": 2881, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3313265809999848, "nonpadding_tokens": 4402, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033861099996101984, "prefetch": {"data_wait_seconds": 0.025994918999415972, "consumed_batches": 2774, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3324, "tokens": 239644260, "loss_tokens": 86166, "loss": 2.2312451401945084, "lr": 0.00029870613310958817, "optimizer_lrs": [0.01991374220730588, 0.00029870613310958817], "gradient_norm": 0.4804449677467346, "seconds": 0.3213154420000137, "nonpadding_tokens_per_second": 268166.38336353697, "ranks": [{"compute_seconds": 0.31764175500006786, "nonpadding_tokens": 18259, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31641291799996907, "nonpadding_tokens": 5412, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31758812299995043, "nonpadding_tokens": 18950, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31778276900013225, "nonpadding_tokens": 17607, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3123127870001099, "nonpadding_tokens": 7459, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31566539499999635, "nonpadding_tokens": 7267, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3177653340001143, "nonpadding_tokens": 4169, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31902319400001034, "nonpadding_tokens": 7043, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035116200001539255, "prefetch": {"data_wait_seconds": 0.026004270999422374, "consumed_batches": 2775, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3325, "tokens": 239699331, "loss_tokens": 55071, "loss": 2.683846194004104, "lr": 0.0002987054852495765, "optimizer_lrs": [0.019913699016638434, 0.0002987054852495765], "gradient_norm": 0.4511711001396179, "seconds": 0.25433160200009297, "nonpadding_tokens_per_second": 216532.27348436185, "ranks": [{"compute_seconds": 0.2516047599999638, "nonpadding_tokens": 4083, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.249391118000176, "nonpadding_tokens": 9957, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2519902040000943, "nonpadding_tokens": 3660, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24903243099993233, "nonpadding_tokens": 9280, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2478234939999311, "nonpadding_tokens": 6041, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25021636499991473, "nonpadding_tokens": 3616, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24783298499983175, "nonpadding_tokens": 14134, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24793651100003444, "nonpadding_tokens": 4300, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039053600016814016, "prefetch": {"data_wait_seconds": 0.026014375999423578, "consumed_batches": 2776, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3326, "tokens": 239806346, "loss_tokens": 107015, "loss": 2.461656660281269, "lr": 0.00029870422585437514, "optimizer_lrs": [0.019913615056958343, 0.00029870422585437514], "gradient_norm": 0.4120394289493561, "seconds": 0.3943872750000992, "nonpadding_tokens_per_second": 271344.9616242641, "ranks": [{"compute_seconds": 0.3823643289999836, "nonpadding_tokens": 11045, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3780642650001482, "nonpadding_tokens": 19431, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3748728100001699, "nonpadding_tokens": 15505, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.37610854000013205, "nonpadding_tokens": 10038, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.38157038700001067, "nonpadding_tokens": 17653, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3788402180000503, "nonpadding_tokens": 21675, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3814573310000924, "nonpadding_tokens": 5836, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.374948293999978, "nonpadding_tokens": 5832, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.009708876999866334, "prefetch": {"data_wait_seconds": 0.026022392999223598, "consumed_batches": 2777, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3327, "tokens": 239879849, "loss_tokens": 73503, "loss": 2.220509069357713, "lr": 0.0002987033604887494, "optimizer_lrs": [0.01991355736591663, 0.0002987033604887494], "gradient_norm": 1.0626574754714966, "seconds": 0.2702914219999002, "nonpadding_tokens_per_second": 271939.8176092586, "ranks": [{"compute_seconds": 0.26592450599991935, "nonpadding_tokens": 5032, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26425803899996936, "nonpadding_tokens": 15762, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.262020406000147, "nonpadding_tokens": 7388, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26186932699988574, "nonpadding_tokens": 4377, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26714078199984215, "nonpadding_tokens": 5627, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2644039790000079, "nonpadding_tokens": 5264, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2604431670001759, "nonpadding_tokens": 20317, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2677518069999678, "nonpadding_tokens": 9736, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004287020001356723, "prefetch": {"data_wait_seconds": 0.026031402999208808, "consumed_batches": 2778, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3328, "tokens": 239933732, "loss_tokens": 53883, "loss": 2.234130256760017, "lr": 0.0002987027259307249, "optimizer_lrs": [0.019913515062048328, 0.0002987027259307249], "gradient_norm": 0.46319350600242615, "seconds": 0.2598016970000572, "nonpadding_tokens_per_second": 207400.4928458498, "ranks": [{"compute_seconds": 0.24024802899998576, "nonpadding_tokens": 7572, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2530385119998755, "nonpadding_tokens": 3492, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2548457729999427, "nonpadding_tokens": 9038, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25028090999990127, "nonpadding_tokens": 16605, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2570899280001413, "nonpadding_tokens": 2116, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2532066339999801, "nonpadding_tokens": 9265, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2491897689999405, "nonpadding_tokens": 2505, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2564375009999367, "nonpadding_tokens": 3290, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004221829999551119, "prefetch": {"data_wait_seconds": 0.026042030999136045, "consumed_batches": 2779, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3329, "tokens": 239991560, "loss_tokens": 57828, "loss": 1.842139891142699, "lr": 0.00029870204474218687, "optimizer_lrs": [0.019913469649479126, 0.00029870204474218687], "gradient_norm": 1.1266545057296753, "seconds": 0.2447517840000728, "nonpadding_tokens_per_second": 236272.02651966288, "ranks": [{"compute_seconds": 0.2386302060001526, "nonpadding_tokens": 4327, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23907654300001013, "nonpadding_tokens": 6006, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2367610720000357, "nonpadding_tokens": 17934, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23709192299997994, "nonpadding_tokens": 5744, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24229699500006063, "nonpadding_tokens": 3881, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2396090440001899, "nonpadding_tokens": 6510, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2351209230000677, "nonpadding_tokens": 3000, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2365533949998735, "nonpadding_tokens": 10426, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034622800012584776, "prefetch": {"data_wait_seconds": 0.02604983099895719, "consumed_batches": 2780, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3330, "tokens": 240047168, "loss_tokens": 55608, "loss": 2.075398323982161, "lr": 0.00029870138953645996, "optimizer_lrs": [0.019913425969097332, 0.00029870138953645996], "gradient_norm": 0.4777156710624695, "seconds": 0.2612070140000924, "nonpadding_tokens_per_second": 212888.61714862043, "ranks": [{"compute_seconds": 0.25452354999993076, "nonpadding_tokens": 5828, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25490090599987525, "nonpadding_tokens": 1853, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2522030539998923, "nonpadding_tokens": 13815, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25754711499985206, "nonpadding_tokens": 12299, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25107365600001685, "nonpadding_tokens": 5761, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2544518130000597, "nonpadding_tokens": 2288, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25069896699983474, "nonpadding_tokens": 11167, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2588203489999614, "nonpadding_tokens": 2597, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033870400011437596, "prefetch": {"data_wait_seconds": 0.026058411998931774, "consumed_batches": 2781, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3331, "tokens": 240145124, "loss_tokens": 97956, "loss": 2.2323591076605824, "lr": 0.0002987002349618307, "optimizer_lrs": [0.01991334899745538, 0.0002987002349618307], "gradient_norm": 0.46259260177612305, "seconds": 0.3565480319998642, "nonpadding_tokens_per_second": 274734.37295549933, "ranks": [{"compute_seconds": 0.34949031799988006, "nonpadding_tokens": 12547, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.350435976999961, "nonpadding_tokens": 11600, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.34765615699984664, "nonpadding_tokens": 17793, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3540121569999428, "nonpadding_tokens": 10951, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3467824419999488, "nonpadding_tokens": 9541, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3497412849999364, "nonpadding_tokens": 18470, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34645481300003667, "nonpadding_tokens": 3195, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3476778149999973, "nonpadding_tokens": 13859, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039182999989861855, "prefetch": {"data_wait_seconds": 0.026066234998964433, "consumed_batches": 2782, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3332, "tokens": 240183368, "loss_tokens": 38244, "loss": 2.5319061486769168, "lr": 0.0002986997840540112, "optimizer_lrs": [0.01991331893693408, 0.0002986997840540112], "gradient_norm": 0.4627540409564972, "seconds": 0.2946385959999134, "nonpadding_tokens_per_second": 129799.69535291719, "ranks": [{"compute_seconds": 0.2907378519998929, "nonpadding_tokens": 5922, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2896202250001352, "nonpadding_tokens": 3166, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2872556049999275, "nonpadding_tokens": 8245, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28813867600001686, "nonpadding_tokens": 9228, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28626029300016853, "nonpadding_tokens": 3026, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.288885670000127, "nonpadding_tokens": 2068, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2860458440000002, "nonpadding_tokens": 2589, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2868194970001241, "nonpadding_tokens": 4000, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004377570000997366, "prefetch": {"data_wait_seconds": 0.026075807998907408, "consumed_batches": 2783, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3333, "tokens": 240228692, "loss_tokens": 45324, "loss": 2.2438158798318772, "lr": 0.0002986992495701794, "optimizer_lrs": [0.019913283304678626, 0.0002986992495701794], "gradient_norm": 0.43003928661346436, "seconds": 0.24012221299994962, "nonpadding_tokens_per_second": 188753.8825906519, "ranks": [{"compute_seconds": 0.23556175399994572, "nonpadding_tokens": 5577, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23382630300011442, "nonpadding_tokens": 4864, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23156978399993022, "nonpadding_tokens": 2323, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23765222800011543, "nonpadding_tokens": 6405, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23768227599998681, "nonpadding_tokens": 5629, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23315854400016178, "nonpadding_tokens": 3994, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23737193499982823, "nonpadding_tokens": 4385, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2312672250000105, "nonpadding_tokens": 12147, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004329599998982303, "prefetch": {"data_wait_seconds": 0.02608519399878162, "consumed_batches": 2784, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3334, "tokens": 240267323, "loss_tokens": 38631, "loss": 2.2185388677486992, "lr": 0.0002986987939273504, "optimizer_lrs": [0.01991325292849003, 0.0002986987939273504], "gradient_norm": 0.45408132672309875, "seconds": 0.2986472570000842, "nonpadding_tokens_per_second": 129353.27244605869, "ranks": [{"compute_seconds": 0.2954801350001617, "nonpadding_tokens": 5485, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2924123719999443, "nonpadding_tokens": 5994, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2909127890000036, "nonpadding_tokens": 4909, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2956541070000185, "nonpadding_tokens": 4295, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2961551959999724, "nonpadding_tokens": 2106, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2925375500001337, "nonpadding_tokens": 1957, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29552755600002456, "nonpadding_tokens": 3000, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2903830800000833, "nonpadding_tokens": 10885, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036188100011713686, "prefetch": {"data_wait_seconds": 0.02609397799865576, "consumed_batches": 2785, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3335, "tokens": 240336392, "loss_tokens": 69069, "loss": 2.1398705461205463, "lr": 0.00029869797907814983, "optimizer_lrs": [0.01991319860520999, 0.00029869797907814983], "gradient_norm": 0.5028471350669861, "seconds": 0.371737807000045, "nonpadding_tokens_per_second": 185800.31059362128, "ranks": [{"compute_seconds": 0.3669733300000644, "nonpadding_tokens": 11954, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3666950250001264, "nonpadding_tokens": 3945, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36346887200011224, "nonpadding_tokens": 11574, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.36443537199988896, "nonpadding_tokens": 4034, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3633442010000181, "nonpadding_tokens": 7871, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3669124279999778, "nonpadding_tokens": 10523, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3692127089998394, "nonpadding_tokens": 14726, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3636249830001361, "nonpadding_tokens": 4442, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000421727999992072, "prefetch": {"data_wait_seconds": 0.026105790998784073, "consumed_batches": 2786, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3336, "tokens": 240404969, "loss_tokens": 68577, "loss": 2.512875125771031, "lr": 0.00029869716978225575, "optimizer_lrs": [0.019913144652150383, 0.00029869716978225575], "gradient_norm": 0.44122564792633057, "seconds": 0.2723976209999819, "nonpadding_tokens_per_second": 251753.30000405753, "ranks": [{"compute_seconds": 0.2624457209999491, "nonpadding_tokens": 2453, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2651588950000132, "nonpadding_tokens": 4801, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26949260900005356, "nonpadding_tokens": 4594, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2696885160000875, "nonpadding_tokens": 4813, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26048422100006974, "nonpadding_tokens": 6292, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26407574099994235, "nonpadding_tokens": 10069, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2697473050000099, "nonpadding_tokens": 9264, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26128038700016987, "nonpadding_tokens": 26291, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036881500000163214, "prefetch": {"data_wait_seconds": 0.026114649998774553, "consumed_batches": 2787, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3337, "tokens": 240453365, "loss_tokens": 48396, "loss": 2.115082793515993, "lr": 0.0002986965984972495, "optimizer_lrs": [0.019913106566483305, 0.0002986965984972495], "gradient_norm": 0.517734706401825, "seconds": 0.2762866129999111, "nonpadding_tokens_per_second": 175165.9245249627, "ranks": [{"compute_seconds": 0.27258390399993004, "nonpadding_tokens": 7040, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.269976958000143, "nonpadding_tokens": 3145, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2722127500001079, "nonpadding_tokens": 6946, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2729288709999764, "nonpadding_tokens": 11980, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2737189510000917, "nonpadding_tokens": 5196, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2691295539998464, "nonpadding_tokens": 5782, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2660230629999205, "nonpadding_tokens": 3103, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26739114899987726, "nonpadding_tokens": 5204, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00042490700002417725, "prefetch": {"data_wait_seconds": 0.026123783998812655, "consumed_batches": 2788, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3338, "tokens": 240516083, "loss_tokens": 62718, "loss": 2.30132522760611, "lr": 0.0002986958579644518, "optimizer_lrs": [0.019913057197630122, 0.0002986958579644518], "gradient_norm": 0.413803368806839, "seconds": 0.32479825699988396, "nonpadding_tokens_per_second": 193098.3268793293, "ranks": [{"compute_seconds": 0.32225115100004587, "nonpadding_tokens": 7251, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31962092399999165, "nonpadding_tokens": 10236, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31652990600014164, "nonpadding_tokens": 6539, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3169659430000138, "nonpadding_tokens": 6693, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31573972699993647, "nonpadding_tokens": 3807, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31987793500002226, "nonpadding_tokens": 11355, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.321932876000119, "nonpadding_tokens": 12415, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31599735199984025, "nonpadding_tokens": 4422, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004073939999216236, "prefetch": {"data_wait_seconds": 0.026132812998824306, "consumed_batches": 2789, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3339, "tokens": 240611968, "loss_tokens": 95885, "loss": 2.3094240496428013, "lr": 0.00029869472541301596, "optimizer_lrs": [0.019912981694201064, 0.00029869472541301596], "gradient_norm": 0.3570566773414612, "seconds": 0.33178600199994435, "nonpadding_tokens_per_second": 288996.520112431, "ranks": [{"compute_seconds": 0.3272756260000733, "nonpadding_tokens": 12365, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3253228250000575, "nonpadding_tokens": 16777, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.327909118999969, "nonpadding_tokens": 2852, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3233199840001362, "nonpadding_tokens": 8231, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.32897887299986905, "nonpadding_tokens": 13868, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32639584300000024, "nonpadding_tokens": 19488, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32143827100003364, "nonpadding_tokens": 16958, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32927729099992575, "nonpadding_tokens": 5346, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040898199995353934, "prefetch": {"data_wait_seconds": 0.02614216899901578, "consumed_batches": 2790, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3340, "tokens": 240667663, "loss_tokens": 55695, "loss": 2.224087805233863, "lr": 0.0002986940673435921, "optimizer_lrs": [0.01991293782290614, 0.0002986940673435921], "gradient_norm": 0.3796444833278656, "seconds": 0.2853866310001649, "nonpadding_tokens_per_second": 195156.3035900158, "ranks": [{"compute_seconds": 0.28297150799994597, "nonpadding_tokens": 8495, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2802111270000296, "nonpadding_tokens": 4277, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.281661387999975, "nonpadding_tokens": 5644, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27763501999993423, "nonpadding_tokens": 4795, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2758047269999224, "nonpadding_tokens": 19660, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2786196419999669, "nonpadding_tokens": 3337, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2760573830000794, "nonpadding_tokens": 4243, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2762685130001046, "nonpadding_tokens": 5244, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003855959998873004, "prefetch": {"data_wait_seconds": 0.026150694999117263, "consumed_batches": 2791, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3341, "tokens": 240717951, "loss_tokens": 50288, "loss": 2.186397599825008, "lr": 0.00029869347301929387, "optimizer_lrs": [0.01991289820128626, 0.00029869347301929387], "gradient_norm": 0.37722840905189514, "seconds": 0.2976630400000886, "nonpadding_tokens_per_second": 168942.7078349567, "ranks": [{"compute_seconds": 0.28965386499999113, "nonpadding_tokens": 2477, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2919485080001323, "nonpadding_tokens": 10690, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2939451830000053, "nonpadding_tokens": 5412, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2892579350000233, "nonpadding_tokens": 12467, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29518522299986216, "nonpadding_tokens": 3619, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29107043299995894, "nonpadding_tokens": 3683, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2947167070001342, "nonpadding_tokens": 4214, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2888828749998993, "nonpadding_tokens": 7726, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003876430000673281, "prefetch": {"data_wait_seconds": 0.026160455999118426, "consumed_batches": 2792, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3342, "tokens": 240785948, "loss_tokens": 67997, "loss": 2.208279409385708, "lr": 0.0002986926691887753, "optimizer_lrs": [0.01991284461258502, 0.0002986926691887753], "gradient_norm": 0.762886106967926, "seconds": 0.33053085500000634, "nonpadding_tokens_per_second": 205720.58242489555, "ranks": [{"compute_seconds": 0.32482113200012463, "nonpadding_tokens": 16780, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3247204080000756, "nonpadding_tokens": 15704, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32275496299985207, "nonpadding_tokens": 4539, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32811063999997714, "nonpadding_tokens": 5979, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3210569140001098, "nonpadding_tokens": 2868, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3244376459999785, "nonpadding_tokens": 7235, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3212464440000531, "nonpadding_tokens": 4855, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32210408599985385, "nonpadding_tokens": 10037, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038612799994552915, "prefetch": {"data_wait_seconds": 0.026168728999167, "consumed_batches": 2793, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3343, "tokens": 240843779, "loss_tokens": 57831, "loss": 1.9895578009631512, "lr": 0.00029869198534264284, "optimizer_lrs": [0.019912799022842858, 0.00029869198534264284], "gradient_norm": 0.5574579834938049, "seconds": 0.31549890399992364, "nonpadding_tokens_per_second": 183300.16132168242, "ranks": [{"compute_seconds": 0.29870199499987393, "nonpadding_tokens": 14062, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3098927549999644, "nonpadding_tokens": 11660, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30638894000003347, "nonpadding_tokens": 5880, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3119709310001326, "nonpadding_tokens": 9627, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30579740899997887, "nonpadding_tokens": 2996, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3078527419997954, "nonpadding_tokens": 6277, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3123233770002116, "nonpadding_tokens": 4102, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3130441589999009, "nonpadding_tokens": 3227, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000399712999978874, "prefetch": {"data_wait_seconds": 0.026176957998998773, "consumed_batches": 2794, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3344, "tokens": 240900920, "loss_tokens": 57141, "loss": 2.042619080432614, "lr": 0.0002986913094809296, "optimizer_lrs": [0.01991275396539531, 0.0002986913094809296], "gradient_norm": 0.465309202671051, "seconds": 0.26483776999998554, "nonpadding_tokens_per_second": 215758.50000550572, "ranks": [{"compute_seconds": 0.2566693230000965, "nonpadding_tokens": 5005, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25810800499994, "nonpadding_tokens": 5564, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25513057099988146, "nonpadding_tokens": 6620, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26100227599999926, "nonpadding_tokens": 11009, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2543462560001899, "nonpadding_tokens": 9250, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2567662310000287, "nonpadding_tokens": 9182, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2538228989999425, "nonpadding_tokens": 5517, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26230602700002237, "nonpadding_tokens": 4994, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003805789999660192, "prefetch": {"data_wait_seconds": 0.026185581998973362, "consumed_batches": 2795, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3345, "tokens": 240978781, "loss_tokens": 77861, "loss": 2.4209733531549813, "lr": 0.00029869038826417054, "optimizer_lrs": [0.019912692550944706, 0.00029869038826417054], "gradient_norm": 0.38327962160110474, "seconds": 0.29627077399982227, "nonpadding_tokens_per_second": 262803.51230339956, "ranks": [{"compute_seconds": 0.2937361469998905, "nonpadding_tokens": 8421, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29040228700000625, "nonpadding_tokens": 8730, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2874258589999954, "nonpadding_tokens": 18567, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2930162340001061, "nonpadding_tokens": 9694, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2862311609999324, "nonpadding_tokens": 12348, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2894799690000127, "nonpadding_tokens": 3972, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2862537689998135, "nonpadding_tokens": 12178, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28707528900008583, "nonpadding_tokens": 3951, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003849750000881613, "prefetch": {"data_wait_seconds": 0.026193596999021906, "consumed_batches": 2796, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3346, "tokens": 241047806, "loss_tokens": 69025, "loss": 2.0951672854038392, "lr": 0.000298689571321327, "optimizer_lrs": [0.019912638088088467, 0.000298689571321327], "gradient_norm": 0.4403793215751648, "seconds": 0.28599208800005727, "nonpadding_tokens_per_second": 241352.83071182788, "ranks": [{"compute_seconds": 0.28213267699993594, "nonpadding_tokens": 4088, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2785653009998441, "nonpadding_tokens": 14976, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2761833250001473, "nonpadding_tokens": 13072, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2767662339999788, "nonpadding_tokens": 6404, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28260324200005016, "nonpadding_tokens": 3379, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2780463689998669, "nonpadding_tokens": 9959, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27511137999999846, "nonpadding_tokens": 6965, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2833335590000843, "nonpadding_tokens": 10182, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00044334899985187803, "prefetch": {"data_wait_seconds": 0.02620447799900205, "consumed_batches": 2797, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3347, "tokens": 241103234, "loss_tokens": 55428, "loss": 2.2458203096810276, "lr": 0.0002986889151217727, "optimizer_lrs": [0.019912594341451515, 0.0002986889151217727], "gradient_norm": 0.4103968143463135, "seconds": 0.27824805299997024, "nonpadding_tokens_per_second": 199203.55022216786, "ranks": [{"compute_seconds": 0.2754953269998168, "nonpadding_tokens": 4068, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2710272689998874, "nonpadding_tokens": 8898, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26806574300007924, "nonpadding_tokens": 8993, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.274319604999846, "nonpadding_tokens": 9655, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2753173119999701, "nonpadding_tokens": 8630, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27068295999993097, "nonpadding_tokens": 2466, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26688371700015523, "nonpadding_tokens": 9875, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26798086200005855, "nonpadding_tokens": 2843, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043553199998314085, "prefetch": {"data_wait_seconds": 0.026214131998997914, "consumed_batches": 2798, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3348, "tokens": 241153935, "loss_tokens": 50701, "loss": 2.2957778815999683, "lr": 0.0002986883147409635, "optimizer_lrs": [0.019912554316064236, 0.0002986883147409635], "gradient_norm": 0.4212639629840851, "seconds": 0.29422903599993333, "nonpadding_tokens_per_second": 172318.13926077468, "ranks": [{"compute_seconds": 0.2864921030000005, "nonpadding_tokens": 7916, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2882831350000288, "nonpadding_tokens": 6263, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28509026300002915, "nonpadding_tokens": 5655, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2854792999999063, "nonpadding_tokens": 10111, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29104053400010343, "nonpadding_tokens": 5541, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28661237899996195, "nonpadding_tokens": 6570, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28433933600013006, "nonpadding_tokens": 1757, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29146383000011156, "nonpadding_tokens": 6888, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045091699985277955, "prefetch": {"data_wait_seconds": 0.026224387998993848, "consumed_batches": 2799, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3349, "tokens": 241215655, "loss_tokens": 61720, "loss": 2.2166204228775115, "lr": 0.0002986875836930199, "optimizer_lrs": [0.019912505579534664, 0.0002986875836930199], "gradient_norm": 0.40618324279785156, "seconds": 0.2997998499999994, "nonpadding_tokens_per_second": 205870.68339093606, "ranks": [{"compute_seconds": 0.29184748999978183, "nonpadding_tokens": 6487, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2943243250001615, "nonpadding_tokens": 7128, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29031875199984825, "nonpadding_tokens": 9899, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29571071000009397, "nonpadding_tokens": 9779, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29661817700002757, "nonpadding_tokens": 6362, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29329509599983794, "nonpadding_tokens": 2074, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2962401359998239, "nonpadding_tokens": 11049, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.297044102999962, "nonpadding_tokens": 8942, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043337900001461094, "prefetch": {"data_wait_seconds": 0.026234915998841757, "consumed_batches": 2800, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3350, "tokens": 241275307, "loss_tokens": 59652, "loss": 2.0978128876651243, "lr": 0.0002986868769470881, "optimizer_lrs": [0.019912458463139205, 0.0002986868769470881], "gradient_norm": 0.47457560896873474, "seconds": 0.34635035699989203, "nonpadding_tokens_per_second": 172230.2252456422, "ranks": [{"compute_seconds": 0.3373350510000819, "nonpadding_tokens": 12328, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3375936479999382, "nonpadding_tokens": 10895, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33594170300011683, "nonpadding_tokens": 4254, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33709463400009554, "nonpadding_tokens": 1349, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33512453000003006, "nonpadding_tokens": 2723, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3381856229998448, "nonpadding_tokens": 10405, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3414888909999263, "nonpadding_tokens": 12012, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.34381420900012927, "nonpadding_tokens": 5686, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034519099995122815, "prefetch": {"data_wait_seconds": 0.02624351799882163, "consumed_batches": 2801, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3351, "tokens": 241346180, "loss_tokens": 70873, "loss": 2.2848980306322577, "lr": 0.00029868603701073193, "optimizer_lrs": [0.01991240246738213, 0.00029868603701073193], "gradient_norm": 0.3828459680080414, "seconds": 0.27757557599989013, "nonpadding_tokens_per_second": 255328.66047273573, "ranks": [{"compute_seconds": 0.2751783159999377, "nonpadding_tokens": 4061, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27347975299994687, "nonpadding_tokens": 9185, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27419875300006424, "nonpadding_tokens": 13206, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27148212599990984, "nonpadding_tokens": 4554, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2695382859999427, "nonpadding_tokens": 14810, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2722715520001202, "nonpadding_tokens": 5118, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2700061029997869, "nonpadding_tokens": 15260, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27030236299992794, "nonpadding_tokens": 4679, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003842200001145102, "prefetch": {"data_wait_seconds": 0.02625273499889147, "consumed_batches": 2802, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3352, "tokens": 241401296, "loss_tokens": 55116, "loss": 2.173781773895058, "lr": 0.0002986853836304083, "optimizer_lrs": [0.019912358908693886, 0.0002986853836304083], "gradient_norm": 0.4107024669647217, "seconds": 0.24785096599998724, "nonpadding_tokens_per_second": 222375.57064838245, "ranks": [{"compute_seconds": 0.24494947300013337, "nonpadding_tokens": 11492, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2428318350000609, "nonpadding_tokens": 8093, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2446558090000508, "nonpadding_tokens": 3526, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24092945999996118, "nonpadding_tokens": 5263, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2391517590001513, "nonpadding_tokens": 3526, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24329630900001575, "nonpadding_tokens": 4576, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24543260399991595, "nonpadding_tokens": 11501, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23985773799995513, "nonpadding_tokens": 7139, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037988000008226663, "prefetch": {"data_wait_seconds": 0.02626260899883448, "consumed_batches": 2803, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3353, "tokens": 241461271, "loss_tokens": 59975, "loss": 2.5229058982909547, "lr": 0.00029868467246477583, "optimizer_lrs": [0.019912311497651724, 0.00029868467246477583], "gradient_norm": 0.46771517395973206, "seconds": 0.27103245299986156, "nonpadding_tokens_per_second": 221283.46379254677, "ranks": [{"compute_seconds": 0.26864901999988433, "nonpadding_tokens": 2403, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2641266790001282, "nonpadding_tokens": 5381, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26144759399994655, "nonpadding_tokens": 3661, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2681776679999075, "nonpadding_tokens": 8617, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26049954799987063, "nonpadding_tokens": 7172, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26647281600003225, "nonpadding_tokens": 17444, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2683531640000183, "nonpadding_tokens": 8255, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26108919199987213, "nonpadding_tokens": 7042, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035912300018026144, "prefetch": {"data_wait_seconds": 0.02627180599866108, "consumed_batches": 2804, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3354, "tokens": 241523771, "loss_tokens": 62500, "loss": 2.168339, "lr": 0.00029868393115482783, "optimizer_lrs": [0.019912262076988523, 0.00029868393115482783], "gradient_norm": 0.5267314314842224, "seconds": 0.30069834300002185, "nonpadding_tokens_per_second": 207849.4991906073, "ranks": [{"compute_seconds": 0.2945248349999474, "nonpadding_tokens": 6471, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2935114029999113, "nonpadding_tokens": 11510, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2961808080001447, "nonpadding_tokens": 6999, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29114031499989323, "nonpadding_tokens": 1922, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2982352379999611, "nonpadding_tokens": 13932, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29178731599995444, "nonpadding_tokens": 2223, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29678939699988405, "nonpadding_tokens": 3549, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28986056600001575, "nonpadding_tokens": 15894, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003573799999685434, "prefetch": {"data_wait_seconds": 0.026280526998746723, "consumed_batches": 2805, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3355, "tokens": 241579748, "loss_tokens": 55977, "loss": 2.540051047751755, "lr": 0.0002986832670375022, "optimizer_lrs": [0.01991221780250015, 0.0002986832670375022], "gradient_norm": 0.43964797258377075, "seconds": 0.30937770300010925, "nonpadding_tokens_per_second": 180934.17675927418, "ranks": [{"compute_seconds": 0.30636534399991433, "nonpadding_tokens": 4787, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3055700060001527, "nonpadding_tokens": 2530, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30671710699994037, "nonpadding_tokens": 6422, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30304719500009014, "nonpadding_tokens": 5042, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30168408199983787, "nonpadding_tokens": 7538, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30436861299995144, "nonpadding_tokens": 12237, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3068238010000641, "nonpadding_tokens": 5243, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3025071140000364, "nonpadding_tokens": 12178, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004218839999339252, "prefetch": {"data_wait_seconds": 0.026290144998668552, "consumed_batches": 2806, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3356, "tokens": 241615105, "loss_tokens": 35357, "loss": 2.5774101928189608, "lr": 0.00029868284747223885, "optimizer_lrs": [0.01991218983148259, 0.00029868284747223885], "gradient_norm": 0.48923948407173157, "seconds": 0.38930576899997504, "nonpadding_tokens_per_second": 90820.64232138894, "ranks": [{"compute_seconds": 0.3855275010000696, "nonpadding_tokens": 3107, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3846423939999113, "nonpadding_tokens": 1536, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3857996690001073, "nonpadding_tokens": 2411, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.38289892800003145, "nonpadding_tokens": 8480, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.38667473900000005, "nonpadding_tokens": 4988, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38383043399994676, "nonpadding_tokens": 5325, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.38155642800006717, "nonpadding_tokens": 6686, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3820671559999482, "nonpadding_tokens": 2824, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041428999998061045, "prefetch": {"data_wait_seconds": 0.026299907998648564, "consumed_batches": 2807, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3357, "tokens": 241676432, "loss_tokens": 61327, "loss": 2.4947810507606762, "lr": 0.00029868211957520335, "optimizer_lrs": [0.019912141305013557, 0.00029868211957520335], "gradient_norm": 0.5332263708114624, "seconds": 0.2622857430001204, "nonpadding_tokens_per_second": 233817.51252858544, "ranks": [{"compute_seconds": 0.2586949959998037, "nonpadding_tokens": 8933, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2565920190002089, "nonpadding_tokens": 2711, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25330710200000794, "nonpadding_tokens": 7836, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2592423889998372, "nonpadding_tokens": 5053, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2523042410000471, "nonpadding_tokens": 10101, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2552705400000832, "nonpadding_tokens": 5212, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2592816799999582, "nonpadding_tokens": 9697, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25981158200011123, "nonpadding_tokens": 11784, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003536280000844272, "prefetch": {"data_wait_seconds": 0.026310064998781968, "consumed_batches": 2808, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3358, "tokens": 241743167, "loss_tokens": 66735, "loss": 2.2621712744436953, "lr": 0.000298681327262679, "optimizer_lrs": [0.0199120884841786, 0.000298681327262679], "gradient_norm": 0.4107857644557953, "seconds": 0.32091112099988095, "nonpadding_tokens_per_second": 207954.77511676747, "ranks": [{"compute_seconds": 0.3142209449999882, "nonpadding_tokens": 7275, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3161156769999707, "nonpadding_tokens": 4152, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31706559899998865, "nonpadding_tokens": 7539, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3137098810000225, "nonpadding_tokens": 3476, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31231000000002496, "nonpadding_tokens": 9618, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31458597000005284, "nonpadding_tokens": 6564, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3118427880001491, "nonpadding_tokens": 13910, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31832502699990073, "nonpadding_tokens": 14201, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003941750001104083, "prefetch": {"data_wait_seconds": 0.02632098499880442, "consumed_batches": 2809, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3359, "tokens": 241822597, "loss_tokens": 79430, "loss": 1.8430914012337907, "lr": 0.00029868038391970886, "optimizer_lrs": [0.01991202559464726, 0.00029868038391970886], "gradient_norm": 0.9264757633209229, "seconds": 0.29415237500006697, "nonpadding_tokens_per_second": 270030.1161939689, "ranks": [{"compute_seconds": 0.29195670699982657, "nonpadding_tokens": 4724, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2910550400001739, "nonpadding_tokens": 22121, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29043357199998354, "nonpadding_tokens": 8339, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29084895300002245, "nonpadding_tokens": 8493, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2871902770000361, "nonpadding_tokens": 3159, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2898200409999845, "nonpadding_tokens": 10205, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29130144600003405, "nonpadding_tokens": 5234, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2877370480000536, "nonpadding_tokens": 17155, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003702060000705387, "prefetch": {"data_wait_seconds": 0.026329975998990562, "consumed_batches": 2810, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3360, "tokens": 241928722, "loss_tokens": 106125, "loss": 2.621138987043581, "lr": 0.00029867912301226275, "optimizer_lrs": [0.01991194153415085, 0.00029867912301226275], "gradient_norm": 0.5933361053466797, "seconds": 0.26390635899997505, "nonpadding_tokens_per_second": 402131.2726307214, "ranks": [{"compute_seconds": 0.26076680899996063, "nonpadding_tokens": 4989, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26112877299988213, "nonpadding_tokens": 4126, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.260189900000114, "nonpadding_tokens": 3138, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2586536710000473, "nonpadding_tokens": 17495, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2615715040001305, "nonpadding_tokens": 19482, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2596074850000605, "nonpadding_tokens": 22119, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2570037840000623, "nonpadding_tokens": 21359, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25746596499993757, "nonpadding_tokens": 13417, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003471509999144473, "prefetch": {"data_wait_seconds": 0.026340135998864866, "consumed_batches": 2811, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3361, "tokens": 241982180, "loss_tokens": 53458, "loss": 2.2475049045512363, "lr": 0.00029867848763254776, "optimizer_lrs": [0.019911899175503185, 0.00029867848763254776], "gradient_norm": 0.5457134246826172, "seconds": 0.24280100400005722, "nonpadding_tokens_per_second": 220172.07144657196, "ranks": [{"compute_seconds": 0.2386183640001036, "nonpadding_tokens": 5329, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2360603990000527, "nonpadding_tokens": 6327, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.234643621000032, "nonpadding_tokens": 9508, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23510102900013408, "nonpadding_tokens": 3817, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24034794299996065, "nonpadding_tokens": 5333, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23637220399996295, "nonpadding_tokens": 10528, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2337880119998772, "nonpadding_tokens": 9613, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23426096000002872, "nonpadding_tokens": 3003, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003535260000262497, "prefetch": {"data_wait_seconds": 0.026349702998871294, "consumed_batches": 2812, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3362, "tokens": 242064309, "loss_tokens": 82129, "loss": 2.2919580020455625, "lr": 0.000298677511184964, "optimizer_lrs": [0.0199118340789976, 0.000298677511184964], "gradient_norm": 0.4174058437347412, "seconds": 0.37274521599988475, "nonpadding_tokens_per_second": 220335.49050305018, "ranks": [{"compute_seconds": 0.3696277579999787, "nonpadding_tokens": 11495, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.36829979599997387, "nonpadding_tokens": 8110, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.36938539299990225, "nonpadding_tokens": 14129, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3700719680000475, "nonpadding_tokens": 11862, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.37030601099991145, "nonpadding_tokens": 6674, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3676812549999795, "nonpadding_tokens": 6059, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3645883320000394, "nonpadding_tokens": 11184, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3651604320000388, "nonpadding_tokens": 12616, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004044040001645044, "prefetch": {"data_wait_seconds": 0.026358448998919926, "consumed_batches": 2813, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3363, "tokens": 242120864, "loss_tokens": 56555, "loss": 2.0638547376447707, "lr": 0.0002986768385829928, "optimizer_lrs": [0.01991178923886619, 0.0002986768385829928], "gradient_norm": 0.43946054577827454, "seconds": 0.31148447900000065, "nonpadding_tokens_per_second": 181566.0291696264, "ranks": [{"compute_seconds": 0.3068682480000007, "nonpadding_tokens": 12498, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3078397809999842, "nonpadding_tokens": 2894, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3051698890001262, "nonpadding_tokens": 4708, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3086089079999965, "nonpadding_tokens": 7522, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3038980599999377, "nonpadding_tokens": 6348, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30632149100006245, "nonpadding_tokens": 7560, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3042073020001226, "nonpadding_tokens": 3110, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30477411899983053, "nonpadding_tokens": 11915, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035392500012676464, "prefetch": {"data_wait_seconds": 0.026366734999058394, "consumed_batches": 2814, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3364, "tokens": 242165624, "loss_tokens": 44760, "loss": 2.230188959171135, "lr": 0.0002986763061369477, "optimizer_lrs": [0.019911753742463182, 0.0002986763061369477], "gradient_norm": 0.6118820905685425, "seconds": 0.2404708080000546, "nonpadding_tokens_per_second": 186134.85924657364, "ranks": [{"compute_seconds": 0.23234972900013418, "nonpadding_tokens": 2556, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2337266100000761, "nonpadding_tokens": 3665, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23228234900011557, "nonpadding_tokens": 4189, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.233239039999944, "nonpadding_tokens": 9464, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23135663299990483, "nonpadding_tokens": 10632, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23380252399988422, "nonpadding_tokens": 7570, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23684115699984432, "nonpadding_tokens": 2042, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23806161900006373, "nonpadding_tokens": 4642, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003741129999070836, "prefetch": {"data_wait_seconds": 0.026375916999043625, "consumed_batches": 2815, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3365, "tokens": 242216589, "loss_tokens": 50965, "loss": 2.2512071703620133, "lr": 0.00029867569974907816, "optimizer_lrs": [0.019911713316605212, 0.00029867569974907816], "gradient_norm": 0.4577026069164276, "seconds": 0.4413002530000085, "nonpadding_tokens_per_second": 115488.26372415204, "ranks": [{"compute_seconds": 0.436911703000078, "nonpadding_tokens": 15027, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.437779651000028, "nonpadding_tokens": 3684, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4347978119999425, "nonpadding_tokens": 2008, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.4349923199999921, "nonpadding_tokens": 3568, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.4389386170000762, "nonpadding_tokens": 7105, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.4366562200000317, "nonpadding_tokens": 10730, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.4335319229999186, "nonpadding_tokens": 3383, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.43885924300002443, "nonpadding_tokens": 5460, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003388169998288504, "prefetch": {"data_wait_seconds": 0.026385331999065897, "consumed_batches": 2816, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3366, "tokens": 242277583, "loss_tokens": 60994, "loss": 2.641319740466275, "lr": 0.0002986749738532878, "optimizer_lrs": [0.01991166492355252, 0.0002986749738532878], "gradient_norm": 0.4522224962711334, "seconds": 0.317998186000068, "nonpadding_tokens_per_second": 191806.12558584518, "ranks": [{"compute_seconds": 0.3154836419998901, "nonpadding_tokens": 11127, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3144054379999943, "nonpadding_tokens": 3302, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31140641199999664, "nonpadding_tokens": 11505, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3120590250000532, "nonpadding_tokens": 9393, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31046323599980497, "nonpadding_tokens": 12780, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3132399339999665, "nonpadding_tokens": 6509, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31000863799999934, "nonpadding_tokens": 2169, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3112369530001615, "nonpadding_tokens": 4209, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040255700014313334, "prefetch": {"data_wait_seconds": 0.026395481999045955, "consumed_batches": 2817, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3367, "tokens": 242354118, "loss_tokens": 76535, "loss": 2.39498370843405, "lr": 0.00029867406272250516, "optimizer_lrs": [0.019911604181500345, 0.00029867406272250516], "gradient_norm": 0.418213814496994, "seconds": 0.3297373279999647, "nonpadding_tokens_per_second": 232108.99555784656, "ranks": [{"compute_seconds": 0.32496787399986715, "nonpadding_tokens": 6542, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.32296509399998286, "nonpadding_tokens": 5373, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.32562689899987163, "nonpadding_tokens": 6580, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3208620980001342, "nonpadding_tokens": 3117, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3200465219999842, "nonpadding_tokens": 12037, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3234424370000397, "nonpadding_tokens": 11890, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3197579569998652, "nonpadding_tokens": 8397, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3272005859998899, "nonpadding_tokens": 22599, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039208300017889997, "prefetch": {"data_wait_seconds": 0.026405381999168753, "consumed_batches": 2818, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3368, "tokens": 242406125, "loss_tokens": 52007, "loss": 1.9656485545215068, "lr": 0.0002986734434138544, "optimizer_lrs": [0.01991156289425696, 0.0002986734434138544], "gradient_norm": 0.42303887009620667, "seconds": 0.3392070679999506, "nonpadding_tokens_per_second": 153319.32882957373, "ranks": [{"compute_seconds": 0.33173678299999665, "nonpadding_tokens": 3280, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3295274120000613, "nonpadding_tokens": 12359, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3323053830001754, "nonpadding_tokens": 1976, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33257274300012796, "nonpadding_tokens": 3942, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3331950589999906, "nonpadding_tokens": 9764, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3292842289999953, "nonpadding_tokens": 5234, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32642567799985045, "nonpadding_tokens": 3892, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33329462100005003, "nonpadding_tokens": 11560, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0037939460000870895, "prefetch": {"data_wait_seconds": 0.026416466999265253, "consumed_batches": 2819, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3369, "tokens": 242532386, "loss_tokens": 126261, "loss": 2.170245067756473, "lr": 0.0002986719392766165, "optimizer_lrs": [0.019911462618441098, 0.0002986719392766165], "gradient_norm": 1.194295048713684, "seconds": 0.35776444900011484, "nonpadding_tokens_per_second": 352916.5638253774, "ranks": [{"compute_seconds": 0.35246578499982206, "nonpadding_tokens": 19119, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3540764179999769, "nonpadding_tokens": 12488, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35371378900003947, "nonpadding_tokens": 7048, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3510950780000712, "nonpadding_tokens": 17300, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.34992203800015886, "nonpadding_tokens": 21091, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3526881609998327, "nonpadding_tokens": 20797, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34988881800018135, "nonpadding_tokens": 11485, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3552539180000167, "nonpadding_tokens": 16933, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003535089999786578, "prefetch": {"data_wait_seconds": 0.026425183999165824, "consumed_batches": 2820, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3370, "tokens": 242623733, "loss_tokens": 91347, "loss": 2.319912155024248, "lr": 0.000298670850538366, "optimizer_lrs": [0.019911390035891065, 0.000298670850538366], "gradient_norm": 0.6547641754150391, "seconds": 0.35723576299983506, "nonpadding_tokens_per_second": 255705.08180067677, "ranks": [{"compute_seconds": 0.35000371799992536, "nonpadding_tokens": 20868, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.35180242599994926, "nonpadding_tokens": 25771, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.35298986399993737, "nonpadding_tokens": 15156, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.35382170299999416, "nonpadding_tokens": 5217, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3544237969999813, "nonpadding_tokens": 4249, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3516051389999575, "nonpadding_tokens": 5649, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.34826191800016204, "nonpadding_tokens": 9116, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3489646470000025, "nonpadding_tokens": 5321, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041405499996471917, "prefetch": {"data_wait_seconds": 0.026435021999262176, "consumed_batches": 2821, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3371, "tokens": 242694685, "loss_tokens": 70952, "loss": 2.0229199846375012, "lr": 0.00029867000457588504, "optimizer_lrs": [0.01991133363839234, 0.00029867000457588504], "gradient_norm": 0.4293883442878723, "seconds": 0.26180395900018993, "nonpadding_tokens_per_second": 271011.9444754024, "ranks": [{"compute_seconds": 0.2568566639999972, "nonpadding_tokens": 9978, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.257252354000002, "nonpadding_tokens": 12113, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25456769900006293, "nonpadding_tokens": 11549, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2551845969999249, "nonpadding_tokens": 15456, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2535248550000233, "nonpadding_tokens": 2455, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2562569770000209, "nonpadding_tokens": 9555, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2533801280001171, "nonpadding_tokens": 6458, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25944154900003014, "nonpadding_tokens": 3388, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003429030000461353, "prefetch": {"data_wait_seconds": 0.026444104999384876, "consumed_batches": 2822, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3372, "tokens": 242774278, "loss_tokens": 79593, "loss": 2.391005843792796, "lr": 0.00029866905526784076, "optimizer_lrs": [0.019911270351189385, 0.00029866905526784076], "gradient_norm": 0.48679453134536743, "seconds": 0.32499926300010884, "nonpadding_tokens_per_second": 244902.09382405074, "ranks": [{"compute_seconds": 0.31917138999983763, "nonpadding_tokens": 3072, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.319116670999847, "nonpadding_tokens": 9017, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.317535141999997, "nonpadding_tokens": 9380, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3222931800000879, "nonpadding_tokens": 11645, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3164881790000891, "nonpadding_tokens": 19282, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3198437250000552, "nonpadding_tokens": 9383, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.322525990000031, "nonpadding_tokens": 6372, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31716604400003234, "nonpadding_tokens": 11442, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000377702999912799, "prefetch": {"data_wait_seconds": 0.026454877999370296, "consumed_batches": 2823, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3373, "tokens": 242850899, "loss_tokens": 76621, "loss": 2.2165539228801503, "lr": 0.000298668141088574, "optimizer_lrs": [0.019911209405904937, 0.000298668141088574], "gradient_norm": 1.6506177186965942, "seconds": 0.3733210230000168, "nonpadding_tokens_per_second": 205241.5890867109, "ranks": [{"compute_seconds": 0.37105502300005355, "nonpadding_tokens": 13095, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3682617530000698, "nonpadding_tokens": 11342, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.364970509999921, "nonpadding_tokens": 12011, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.36585261599998375, "nonpadding_tokens": 9406, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3639537480000854, "nonpadding_tokens": 9668, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3671532670000488, "nonpadding_tokens": 12105, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3702866119999726, "nonpadding_tokens": 6922, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3643727230000877, "nonpadding_tokens": 2072, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034846599987758964, "prefetch": {"data_wait_seconds": 0.026463734999381927, "consumed_batches": 2824, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3374, "tokens": 242918805, "loss_tokens": 67906, "loss": 2.237682881483227, "lr": 0.0002986673306285443, "optimizer_lrs": [0.01991115537523629, 0.0002986673306285443], "gradient_norm": 0.7966649532318115, "seconds": 0.26312480499996127, "nonpadding_tokens_per_second": 258075.2506401287, "ranks": [{"compute_seconds": 0.2232296459999361, "nonpadding_tokens": 7689, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2217868660000022, "nonpadding_tokens": 14777, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21894140599988532, "nonpadding_tokens": 7119, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21908277700003964, "nonpadding_tokens": 10868, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2176011720000588, "nonpadding_tokens": 10863, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2206569130000844, "nonpadding_tokens": 3598, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21795268700020642, "nonpadding_tokens": 9593, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2182596069999363, "nonpadding_tokens": 3399, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.006926733999989665, "prefetch": {"data_wait_seconds": 0.02647188199939876, "consumed_batches": 2825, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3375, "tokens": 242951092, "loss_tokens": 32287, "loss": 2.019472088224363, "lr": 0.0002986669451962463, "optimizer_lrs": [0.019911129679749756, 0.0002986669451962463], "gradient_norm": 0.4410654902458191, "seconds": 0.22252514699994208, "nonpadding_tokens_per_second": 145093.71383544532, "ranks": [{"compute_seconds": 0.21991913299984844, "nonpadding_tokens": 4530, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21804518199996892, "nonpadding_tokens": 1610, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2143971530001636, "nonpadding_tokens": 3196, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21504599099989719, "nonpadding_tokens": 3430, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21362125999985437, "nonpadding_tokens": 2650, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2164179230001082, "nonpadding_tokens": 4695, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21335193400000207, "nonpadding_tokens": 11094, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21445893899999646, "nonpadding_tokens": 1082, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003961649999837391, "prefetch": {"data_wait_seconds": 0.026482408999257245, "consumed_batches": 2826, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3376, "tokens": 243037214, "loss_tokens": 86122, "loss": 2.3340972689905017, "lr": 0.0002986659168270115, "optimizer_lrs": [0.019911061121800768, 0.0002986659168270115], "gradient_norm": 0.4341857135295868, "seconds": 0.2905607189998136, "nonpadding_tokens_per_second": 296399.3216166816, "ranks": [{"compute_seconds": 0.28424532899998667, "nonpadding_tokens": 9015, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28634789500006264, "nonpadding_tokens": 11155, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28308714299987514, "nonpadding_tokens": 6483, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2838589809998666, "nonpadding_tokens": 3098, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28811881999990874, "nonpadding_tokens": 21497, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2853270909999992, "nonpadding_tokens": 2957, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2822612229999777, "nonpadding_tokens": 13275, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2828912320001109, "nonpadding_tokens": 18642, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000403746999836585, "prefetch": {"data_wait_seconds": 0.02649221699921611, "consumed_batches": 2827, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3377, "tokens": 243099016, "loss_tokens": 61802, "loss": 2.327681547522734, "lr": 0.000298665178615916, "optimizer_lrs": [0.019911011907727733, 0.000298665178615916], "gradient_norm": 0.42733514308929443, "seconds": 0.2926189219999742, "nonpadding_tokens_per_second": 211203.01987854857, "ranks": [{"compute_seconds": 0.2893478319999758, "nonpadding_tokens": 4108, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2879830229999243, "nonpadding_tokens": 6733, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2887675399999807, "nonpadding_tokens": 16344, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.288875378000057, "nonpadding_tokens": 7382, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2871171570000115, "nonpadding_tokens": 3294, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29028670100001364, "nonpadding_tokens": 13432, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.287422289999995, "nonpadding_tokens": 6136, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28805235700019693, "nonpadding_tokens": 4373, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003808500000559434, "prefetch": {"data_wait_seconds": 0.02650157799935471, "consumed_batches": 2828, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3378, "tokens": 243168459, "loss_tokens": 69443, "loss": 2.274057500396008, "lr": 0.00029866434889237666, "optimizer_lrs": [0.019910956592825112, 0.00029866434889237666], "gradient_norm": 0.4272650182247162, "seconds": 0.30777425499991296, "nonpadding_tokens_per_second": 225629.6583351965, "ranks": [{"compute_seconds": 0.30255950600007964, "nonpadding_tokens": 13069, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3045724399999017, "nonpadding_tokens": 2912, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30127140600006896, "nonpadding_tokens": 2189, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30447359900017545, "nonpadding_tokens": 21270, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3007696530000885, "nonpadding_tokens": 2797, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3034391379999306, "nonpadding_tokens": 5586, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30075026200006505, "nonpadding_tokens": 15997, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3054584960000284, "nonpadding_tokens": 5623, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033556900007170043, "prefetch": {"data_wait_seconds": 0.026510894999276502, "consumed_batches": 2829, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3379, "tokens": 243204350, "loss_tokens": 35891, "loss": 2.217988144660221, "lr": 0.0002986639199565584, "optimizer_lrs": [0.019910927997103898, 0.0002986639199565584], "gradient_norm": 0.46101170778274536, "seconds": 0.20665609100001348, "nonpadding_tokens_per_second": 173675.0164310311, "ranks": [{"compute_seconds": 0.20373881800014715, "nonpadding_tokens": 2774, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20333281500006706, "nonpadding_tokens": 6036, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20069879500010757, "nonpadding_tokens": 2561, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20127420900007564, "nonpadding_tokens": 1317, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.19986938100009866, "nonpadding_tokens": 4906, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20258800400006294, "nonpadding_tokens": 3354, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20416289899981166, "nonpadding_tokens": 6205, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2043524589998924, "nonpadding_tokens": 8738, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033142000006591843, "prefetch": {"data_wait_seconds": 0.026519048999219308, "consumed_batches": 2830, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3380, "tokens": 243267702, "loss_tokens": 63352, "loss": 2.370400204413436, "lr": 0.00029866316266521486, "optimizer_lrs": [0.019910877511014325, 0.00029866316266521486], "gradient_norm": 0.4274333417415619, "seconds": 0.25281970799983355, "nonpadding_tokens_per_second": 250581.73075669285, "ranks": [{"compute_seconds": 0.2496945529999266, "nonpadding_tokens": 8300, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2469215029998395, "nonpadding_tokens": 8700, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24637316999996983, "nonpadding_tokens": 15238, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2504798130000836, "nonpadding_tokens": 6156, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2456584470000962, "nonpadding_tokens": 2849, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24817974200004755, "nonpadding_tokens": 1090, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25033159600002364, "nonpadding_tokens": 7542, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2456879219998882, "nonpadding_tokens": 13477, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003683369998270791, "prefetch": {"data_wait_seconds": 0.02652905699933683, "consumed_batches": 2831, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3381, "tokens": 243344708, "loss_tokens": 77006, "loss": 1.9205697039841052, "lr": 0.00029866224187047026, "optimizer_lrs": [0.019910816124698017, 0.00029866224187047026], "gradient_norm": 0.46722257137298584, "seconds": 0.2572648590000881, "nonpadding_tokens_per_second": 299325.76217093697, "ranks": [{"compute_seconds": 0.2522958899999139, "nonpadding_tokens": 5161, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2541328089998842, "nonpadding_tokens": 12638, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2512986950000595, "nonpadding_tokens": 20080, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25194600400004674, "nonpadding_tokens": 17225, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25006579499995496, "nonpadding_tokens": 8976, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2528741439998612, "nonpadding_tokens": 6284, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.254080163000026, "nonpadding_tokens": 3227, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2549696609999046, "nonpadding_tokens": 3415, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003344280000874278, "prefetch": {"data_wait_seconds": 0.026537279999232055, "consumed_batches": 2832, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3382, "tokens": 243408197, "loss_tokens": 63489, "loss": 2.2609027252752445, "lr": 0.00029866148246722793, "optimizer_lrs": [0.019910765497815196, 0.00029866148246722793], "gradient_norm": 0.4562474489212036, "seconds": 0.22801433299991913, "nonpadding_tokens_per_second": 278443.0222639667, "ranks": [{"compute_seconds": 0.22324312099999588, "nonpadding_tokens": 4532, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2241040559999874, "nonpadding_tokens": 9938, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22122701700004654, "nonpadding_tokens": 16680, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2245483089998288, "nonpadding_tokens": 15405, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22509811300005822, "nonpadding_tokens": 1764, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2225917959999606, "nonpadding_tokens": 2741, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22011401700001443, "nonpadding_tokens": 6867, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22568547799983207, "nonpadding_tokens": 5562, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037693500007662806, "prefetch": {"data_wait_seconds": 0.026544976999232972, "consumed_batches": 2833, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3383, "tokens": 243467073, "loss_tokens": 58876, "loss": 1.9900569629390583, "lr": 0.0002986607780492929, "optimizer_lrs": [0.01991071853661953, 0.0002986607780492929], "gradient_norm": 0.6534141898155212, "seconds": 0.31250039199994717, "nonpadding_tokens_per_second": 188402.96366735423, "ranks": [{"compute_seconds": 0.308441356000003, "nonpadding_tokens": 20331, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30992701199988915, "nonpadding_tokens": 2951, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3090495579999697, "nonpadding_tokens": 4693, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3074833609998677, "nonpadding_tokens": 4357, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30575716799990005, "nonpadding_tokens": 7426, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30860937099987495, "nonpadding_tokens": 9890, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3056876589998865, "nonpadding_tokens": 5472, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30987408299984054, "nonpadding_tokens": 3756, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033651399985501484, "prefetch": {"data_wait_seconds": 0.0265528629992815, "consumed_batches": 2834, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3384, "tokens": 243521150, "loss_tokens": 54077, "loss": 2.329594836991697, "lr": 0.0002986601308862179, "optimizer_lrs": [0.019910675392414527, 0.0002986601308862179], "gradient_norm": 0.4239283800125122, "seconds": 0.33452130599994234, "nonpadding_tokens_per_second": 161654.875280229, "ranks": [{"compute_seconds": 0.332127360999948, "nonpadding_tokens": 17054, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33040475000007063, "nonpadding_tokens": 3685, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3279796690001149, "nonpadding_tokens": 9594, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3316607900001145, "nonpadding_tokens": 3448, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3268198539999503, "nonpadding_tokens": 8772, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3294101609999416, "nonpadding_tokens": 2406, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33187146199998097, "nonpadding_tokens": 6827, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.332221650000065, "nonpadding_tokens": 2291, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003915659999620402, "prefetch": {"data_wait_seconds": 0.026561561999145056, "consumed_batches": 2835, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3385, "tokens": 243580440, "loss_tokens": 59290, "loss": 2.460800461713611, "lr": 0.00029865942115813094, "optimizer_lrs": [0.019910628077208732, 0.00029865942115813094], "gradient_norm": 0.4339131712913513, "seconds": 0.2897186550001152, "nonpadding_tokens_per_second": 204646.81502810522, "ranks": [{"compute_seconds": 0.2861041890000706, "nonpadding_tokens": 5159, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2865629049999825, "nonpadding_tokens": 12200, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28404522200003157, "nonpadding_tokens": 10752, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28461211900003036, "nonpadding_tokens": 3474, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2870984779999617, "nonpadding_tokens": 6316, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2857639719998133, "nonpadding_tokens": 12166, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28274813099983476, "nonpadding_tokens": 3159, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2873801220000587, "nonpadding_tokens": 6064, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004218300000502495, "prefetch": {"data_wait_seconds": 0.026586965999015888, "consumed_batches": 2836, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3386, "tokens": 243613865, "loss_tokens": 33425, "loss": 2.2642223728496633, "lr": 0.0002986590209633597, "optimizer_lrs": [0.019910601397557313, 0.0002986590209633597], "gradient_norm": 0.46224063634872437, "seconds": 0.2145888410000225, "nonpadding_tokens_per_second": 155762.99235427854, "ranks": [{"compute_seconds": 0.212110722000034, "nonpadding_tokens": 2287, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21179947200016613, "nonpadding_tokens": 4245, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20868736700003865, "nonpadding_tokens": 3488, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2119885360000353, "nonpadding_tokens": 5110, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2074669820001418, "nonpadding_tokens": 2118, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21066248500005713, "nonpadding_tokens": 2508, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2073101299999962, "nonpadding_tokens": 7363, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2123376280001139, "nonpadding_tokens": 6306, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034058200003528327, "prefetch": {"data_wait_seconds": 0.026595132999048587, "consumed_batches": 2837, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3387, "tokens": 243652142, "loss_tokens": 38277, "loss": 2.213435940643206, "lr": 0.00029865856260299965, "optimizer_lrs": [0.019910570840199978, 0.00029865856260299965], "gradient_norm": 0.4732402563095093, "seconds": 0.2186071159999301, "nonpadding_tokens_per_second": 175094.9406423359, "ranks": [{"compute_seconds": 0.2163721980000446, "nonpadding_tokens": 1780, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21492896599988853, "nonpadding_tokens": 5874, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21173405599984108, "nonpadding_tokens": 5273, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2119546179999361, "nonpadding_tokens": 1943, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21054317900006936, "nonpadding_tokens": 2994, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2131912889999512, "nonpadding_tokens": 5646, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2155255710001711, "nonpadding_tokens": 9433, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21136716699993485, "nonpadding_tokens": 5334, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034711400007836346, "prefetch": {"data_wait_seconds": 0.026605125998869994, "consumed_batches": 2838, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3388, "tokens": 243714679, "loss_tokens": 62537, "loss": 2.528198306602491, "lr": 0.0002986578135657217, "optimizer_lrs": [0.01991052090438145, 0.0002986578135657217], "gradient_norm": 0.46696746349334717, "seconds": 0.2789228260000982, "nonpadding_tokens_per_second": 224208.97169591268, "ranks": [{"compute_seconds": 0.27426209800000834, "nonpadding_tokens": 3291, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2758118620001824, "nonpadding_tokens": 5669, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27329067899995607, "nonpadding_tokens": 7670, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27378490799992505, "nonpadding_tokens": 10510, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27663368899993657, "nonpadding_tokens": 3452, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27372726600015085, "nonpadding_tokens": 19283, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2760762440000235, "nonpadding_tokens": 3813, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27331078099996375, "nonpadding_tokens": 8849, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035972500018033315, "prefetch": {"data_wait_seconds": 0.02661336399883396, "consumed_batches": 2839, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3389, "tokens": 243770385, "loss_tokens": 55706, "loss": 1.9268102291494633, "lr": 0.00029865714617160626, "optimizer_lrs": [0.01991047641144042, 0.00029865714617160626], "gradient_norm": 0.4830160439014435, "seconds": 0.29140023399986603, "nonpadding_tokens_per_second": 191166.62754644736, "ranks": [{"compute_seconds": 0.28915853099988453, "nonpadding_tokens": 7567, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2851770329998544, "nonpadding_tokens": 9282, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2858110960000886, "nonpadding_tokens": 5603, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28593986800001403, "nonpadding_tokens": 16605, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2851649050001015, "nonpadding_tokens": 2459, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28627568599995357, "nonpadding_tokens": 3239, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2887593200000538, "nonpadding_tokens": 3996, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28569178599991574, "nonpadding_tokens": 6955, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037002299995947396, "prefetch": {"data_wait_seconds": 0.026623385998803428, "consumed_batches": 2840, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3390, "tokens": 243814008, "loss_tokens": 43623, "loss": 2.204598205075304, "lr": 0.00029865662342445275, "optimizer_lrs": [0.019910441561630184, 0.00029865662342445275], "gradient_norm": 0.4549322724342346, "seconds": 0.2905939429999762, "nonpadding_tokens_per_second": 150116.68705016185, "ranks": [{"compute_seconds": 0.2878406579998227, "nonpadding_tokens": 2602, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28458677399999033, "nonpadding_tokens": 9804, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2845158029999766, "nonpadding_tokens": 2438, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2850123050000093, "nonpadding_tokens": 3249, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2837794649999523, "nonpadding_tokens": 4525, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28660616000001937, "nonpadding_tokens": 8451, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28413529899989953, "nonpadding_tokens": 6654, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2883181079998849, "nonpadding_tokens": 5900, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000342244999956165, "prefetch": {"data_wait_seconds": 0.026631121998846652, "consumed_batches": 2841, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3391, "tokens": 243848053, "loss_tokens": 34045, "loss": 2.4089447330738727, "lr": 0.0002986562153829485, "optimizer_lrs": [0.019910414358863236, 0.0002986562153829485], "gradient_norm": 0.5568086504936218, "seconds": 0.23044696300007672, "nonpadding_tokens_per_second": 147734.6438277369, "ranks": [{"compute_seconds": 0.22627311200017175, "nonpadding_tokens": 2732, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22811608299980435, "nonpadding_tokens": 3539, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22606801899996753, "nonpadding_tokens": 4708, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22569062499997017, "nonpadding_tokens": 8217, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2273027840001305, "nonpadding_tokens": 4721, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22663331800004016, "nonpadding_tokens": 3490, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22700638599985723, "nonpadding_tokens": 2816, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22774881400005142, "nonpadding_tokens": 3822, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003492020000521734, "prefetch": {"data_wait_seconds": 0.026639551998826505, "consumed_batches": 2842, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3392, "tokens": 243895820, "loss_tokens": 47767, "loss": 2.127931384376243, "lr": 0.0002986556427744343, "optimizer_lrs": [0.01991037618496229, 0.0002986556427744343], "gradient_norm": 0.4606491029262543, "seconds": 0.2880477260000589, "nonpadding_tokens_per_second": 165830.15829810867, "ranks": [{"compute_seconds": 0.28385628399996676, "nonpadding_tokens": 4048, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.285776880999947, "nonpadding_tokens": 7007, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.283078119000038, "nonpadding_tokens": 6885, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2835050810001576, "nonpadding_tokens": 3770, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2817327400000522, "nonpadding_tokens": 3641, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28320716600001106, "nonpadding_tokens": 3853, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28176484199980223, "nonpadding_tokens": 13535, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2822618209997927, "nonpadding_tokens": 5028, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034218500013594166, "prefetch": {"data_wait_seconds": 0.02664706999890143, "consumed_batches": 2843, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3393, "tokens": 243966577, "loss_tokens": 70757, "loss": 2.5792540844015432, "lr": 0.0002986547943494889, "optimizer_lrs": [0.019910319623299262, 0.0002986547943494889], "gradient_norm": 0.44984257221221924, "seconds": 0.2935219370001505, "nonpadding_tokens_per_second": 241062.05050004055, "ranks": [{"compute_seconds": 0.2870754380001017, "nonpadding_tokens": 9119, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2908650790000138, "nonpadding_tokens": 20818, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2879541520001112, "nonpadding_tokens": 4258, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2886469439999928, "nonpadding_tokens": 9827, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2866294299999481, "nonpadding_tokens": 10810, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28823318700005984, "nonpadding_tokens": 4354, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.286379449000151, "nonpadding_tokens": 7006, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.291212595999923, "nonpadding_tokens": 4565, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034592099996189063, "prefetch": {"data_wait_seconds": 0.0266563819988761, "consumed_batches": 2844, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3394, "tokens": 244029296, "loss_tokens": 62719, "loss": 2.2988742446467576, "lr": 0.0002986540420830375, "optimizer_lrs": [0.0199102694722025, 0.0002986540420830375], "gradient_norm": 0.4352511465549469, "seconds": 0.3025642140000855, "nonpadding_tokens_per_second": 207291.53382290702, "ranks": [{"compute_seconds": 0.3001860530000613, "nonpadding_tokens": 5566, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2988632049998614, "nonpadding_tokens": 10213, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29574983999987126, "nonpadding_tokens": 6126, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29617347299995345, "nonpadding_tokens": 8654, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2948521119999441, "nonpadding_tokens": 13862, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2978681089998645, "nonpadding_tokens": 10641, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2950886809999247, "nonpadding_tokens": 4441, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2955217910000556, "nonpadding_tokens": 3216, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000401754999984405, "prefetch": {"data_wait_seconds": 0.026664694998956406, "consumed_batches": 2845, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3395, "tokens": 244084569, "loss_tokens": 55273, "loss": 2.6824223626363684, "lr": 0.00029865337895221923, "optimizer_lrs": [0.019910225263481282, 0.00029865337895221923], "gradient_norm": 0.48526281118392944, "seconds": 0.2564639970000826, "nonpadding_tokens_per_second": 215519.52962809903, "ranks": [{"compute_seconds": 0.25361831600002915, "nonpadding_tokens": 4634, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2541973939999025, "nonpadding_tokens": 3796, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2510739290000856, "nonpadding_tokens": 7212, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2512125289999858, "nonpadding_tokens": 11864, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24957098700019742, "nonpadding_tokens": 6614, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25105535800003054, "nonpadding_tokens": 4148, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24987567599987415, "nonpadding_tokens": 9641, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2507234680001602, "nonpadding_tokens": 7364, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038225499997679435, "prefetch": {"data_wait_seconds": 0.026673896998957503, "consumed_batches": 2846, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3396, "tokens": 244142914, "loss_tokens": 58345, "loss": 2.144492217627903, "lr": 0.00029865267878917423, "optimizer_lrs": [0.01991017858594495, 0.00029865267878917423], "gradient_norm": 0.9551799893379211, "seconds": 0.41302462599992396, "nonpadding_tokens_per_second": 141262.76334915377, "ranks": [{"compute_seconds": 0.4086496669999633, "nonpadding_tokens": 4996, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.41074132199992164, "nonpadding_tokens": 23538, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.4097716030000811, "nonpadding_tokens": 4482, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.41017864400009785, "nonpadding_tokens": 2733, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.4062639720000334, "nonpadding_tokens": 4922, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.40784404499981974, "nonpadding_tokens": 3750, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.40668222199997217, "nonpadding_tokens": 3714, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.40702972899998713, "nonpadding_tokens": 10210, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003967869999996765, "prefetch": {"data_wait_seconds": 0.026681717999011312, "consumed_batches": 2847, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3397, "tokens": 244226813, "loss_tokens": 83899, "loss": 1.7063750014898866, "lr": 0.00029865167165065625, "optimizer_lrs": [0.019910111443377083, 0.00029865167165065625], "gradient_norm": 0.3953787386417389, "seconds": 0.2868102900001759, "nonpadding_tokens_per_second": 292524.3721205001, "ranks": [{"compute_seconds": 0.2823204489998261, "nonpadding_tokens": 3989, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28448225400006777, "nonpadding_tokens": 11445, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28136774699987654, "nonpadding_tokens": 6726, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2818939789999604, "nonpadding_tokens": 10101, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2806406679999327, "nonpadding_tokens": 6470, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28324108199990405, "nonpadding_tokens": 16899, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28026977400008946, "nonpadding_tokens": 10620, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2809827230000792, "nonpadding_tokens": 17649, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035198299997318827, "prefetch": {"data_wait_seconds": 0.02669162899901778, "consumed_batches": 2848, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3398, "tokens": 244301577, "loss_tokens": 74764, "loss": 2.192298014418704, "lr": 0.0002986507738548361, "optimizer_lrs": [0.019910051590322407, 0.0002986507738548361], "gradient_norm": 0.3794938325881958, "seconds": 0.28049587800001063, "nonpadding_tokens_per_second": 266542.24130879086, "ranks": [{"compute_seconds": 0.2779785010000069, "nonpadding_tokens": 8445, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27411789799998587, "nonpadding_tokens": 13095, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27533899899981407, "nonpadding_tokens": 6569, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27604123699984484, "nonpadding_tokens": 6492, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2738985389999016, "nonpadding_tokens": 6407, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27529350300005717, "nonpadding_tokens": 9501, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27438313000016024, "nonpadding_tokens": 10191, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2748239879999801, "nonpadding_tokens": 14064, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004710850000719802, "prefetch": {"data_wait_seconds": 0.026702381998987335, "consumed_batches": 2849, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3399, "tokens": 244394389, "loss_tokens": 92812, "loss": 1.9039498192582855, "lr": 0.0002986496589178435, "optimizer_lrs": [0.01990997726118957, 0.0002986496589178435], "gradient_norm": 0.45877552032470703, "seconds": 0.34167633900005967, "nonpadding_tokens_per_second": 271637.1881986941, "ranks": [{"compute_seconds": 0.33771161899994695, "nonpadding_tokens": 10624, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33503357299991876, "nonpadding_tokens": 7365, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3354572089999692, "nonpadding_tokens": 10378, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3362421339998036, "nonpadding_tokens": 13482, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3348257529999046, "nonpadding_tokens": 13468, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3378096149999692, "nonpadding_tokens": 18485, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3347737919998508, "nonpadding_tokens": 9449, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33924613299996054, "nonpadding_tokens": 9561, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00045034799995846697, "prefetch": {"data_wait_seconds": 0.026712619998946252, "consumed_batches": 2850, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3400, "tokens": 244479949, "loss_tokens": 85560, "loss": 2.3225201978143994, "lr": 0.000298648630692207, "optimizer_lrs": [0.0199099087128138, 0.000298648630692207], "gradient_norm": 0.4544578194618225, "seconds": 0.29243625000003703, "nonpadding_tokens_per_second": 292576.5872048666, "ranks": [{"compute_seconds": 0.2898500830001467, "nonpadding_tokens": 19285, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28994964000003165, "nonpadding_tokens": 10983, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2870107010000993, "nonpadding_tokens": 7777, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2894556639998882, "nonpadding_tokens": 2518, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28572890400005235, "nonpadding_tokens": 8465, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2873102819999076, "nonpadding_tokens": 5678, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2856742819999454, "nonpadding_tokens": 14316, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.290099901000076, "nonpadding_tokens": 16538, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003940169999623322, "prefetch": {"data_wait_seconds": 0.026720074999005305, "consumed_batches": 2851, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3401, "tokens": 244558009, "loss_tokens": 78060, "loss": 2.6608594350499617, "lr": 0.00029864769225889417, "optimizer_lrs": [0.019909846150592946, 0.00029864769225889417], "gradient_norm": 0.8409583568572998, "seconds": 0.3418411489999471, "nonpadding_tokens_per_second": 228351.67804801662, "ranks": [{"compute_seconds": 0.33601416600004086, "nonpadding_tokens": 11022, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3359168680001403, "nonpadding_tokens": 6444, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33854151199989246, "nonpadding_tokens": 2934, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.33542106400000193, "nonpadding_tokens": 9569, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.33944208599996273, "nonpadding_tokens": 8689, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3364587619998929, "nonpadding_tokens": 18778, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33348270400006186, "nonpadding_tokens": 17593, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33434549100002187, "nonpadding_tokens": 3031, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003721779999068531, "prefetch": {"data_wait_seconds": 0.026730035999165125, "consumed_batches": 2852, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3402, "tokens": 244627281, "loss_tokens": 69272, "loss": 2.2764221113869962, "lr": 0.00029864685920305523, "optimizer_lrs": [0.019909790613537016, 0.00029864685920305523], "gradient_norm": 0.4742138981819153, "seconds": 0.24728537800001504, "nonpadding_tokens_per_second": 280129.78591882525, "ranks": [{"compute_seconds": 0.2423770840000543, "nonpadding_tokens": 2454, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24455181000007542, "nonpadding_tokens": 10698, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2440117010000904, "nonpadding_tokens": 2115, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2420620840000538, "nonpadding_tokens": 10740, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24091051100003824, "nonpadding_tokens": 4276, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24333988599983059, "nonpadding_tokens": 17271, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24496689300008256, "nonpadding_tokens": 14311, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24128452000013567, "nonpadding_tokens": 7407, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003660139998373779, "prefetch": {"data_wait_seconds": 0.026738338999123812, "consumed_batches": 2853, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3403, "tokens": 244701610, "loss_tokens": 74329, "loss": 2.6248023987945484, "lr": 0.00029864596504856054, "optimizer_lrs": [0.019909731003237372, 0.00029864596504856054], "gradient_norm": 0.47022607922554016, "seconds": 0.28719741600002635, "nonpadding_tokens_per_second": 258808.0388578189, "ranks": [{"compute_seconds": 0.25814901700005066, "nonpadding_tokens": 3843, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26151153099999647, "nonpadding_tokens": 13048, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25814554699991277, "nonpadding_tokens": 7113, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2582960170000206, "nonpadding_tokens": 4260, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25883122100003675, "nonpadding_tokens": 12990, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25931406299991977, "nonpadding_tokens": 10995, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2617939579999984, "nonpadding_tokens": 3387, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26268588600009934, "nonpadding_tokens": 18693, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.010298315000000002, "prefetch": {"data_wait_seconds": 0.026746002999246343, "consumed_batches": 2854, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3404, "tokens": 244760748, "loss_tokens": 59138, "loss": 2.638456343636917, "lr": 0.0002986452534271208, "optimizer_lrs": [0.019909683561808056, 0.0002986452534271208], "gradient_norm": 0.4321073591709137, "seconds": 0.27908800799991695, "nonpadding_tokens_per_second": 211897.31663431987, "ranks": [{"compute_seconds": 0.2752792789999603, "nonpadding_tokens": 13206, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27538819699998385, "nonpadding_tokens": 10448, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2720543220000309, "nonpadding_tokens": 8530, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27629773099988597, "nonpadding_tokens": 13307, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2766655630000514, "nonpadding_tokens": 2039, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2737715550001667, "nonpadding_tokens": 3620, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27159467399997084, "nonpadding_tokens": 1767, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27190472500001306, "nonpadding_tokens": 6221, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039222399982463685, "prefetch": {"data_wait_seconds": 0.026754526999368977, "consumed_batches": 2855, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3405, "tokens": 244833730, "loss_tokens": 72982, "loss": 2.678993193527171, "lr": 0.00029864437496116036, "optimizer_lrs": [0.019909624997410693, 0.00029864437496116036], "gradient_norm": 0.5137705206871033, "seconds": 0.24335758899997018, "nonpadding_tokens_per_second": 299896.13350421935, "ranks": [{"compute_seconds": 0.23965872799999488, "nonpadding_tokens": 5200, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2374869420000323, "nonpadding_tokens": 14074, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24033606300008614, "nonpadding_tokens": 13216, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24098589400000492, "nonpadding_tokens": 6765, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2372489859999405, "nonpadding_tokens": 11670, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24012201799996546, "nonpadding_tokens": 8272, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23698016599996663, "nonpadding_tokens": 4447, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2374648579998393, "nonpadding_tokens": 9338, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003944459999729588, "prefetch": {"data_wait_seconds": 0.026762703999295923, "consumed_batches": 2856, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3406, "tokens": 244886778, "loss_tokens": 53048, "loss": 2.4219233053084, "lr": 0.0002986437362579504, "optimizer_lrs": [0.019909582417196696, 0.0002986437362579504], "gradient_norm": 0.4644162356853485, "seconds": 0.25752851500010365, "nonpadding_tokens_per_second": 205988.8397212194, "ranks": [{"compute_seconds": 0.2547578539999904, "nonpadding_tokens": 2217, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25468204000003425, "nonpadding_tokens": 1861, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25127135500019904, "nonpadding_tokens": 4727, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2522542679998878, "nonpadding_tokens": 3842, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25056948400015244, "nonpadding_tokens": 3701, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2528936059998159, "nonpadding_tokens": 15118, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2544371129999945, "nonpadding_tokens": 10755, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25518044600016765, "nonpadding_tokens": 10827, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033489099996586447, "prefetch": {"data_wait_seconds": 0.026770994999381514, "consumed_batches": 2857, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3407, "tokens": 244954384, "loss_tokens": 67606, "loss": 2.4290939875898587, "lr": 0.00029864292205805524, "optimizer_lrs": [0.019909528137203685, 0.00029864292205805524], "gradient_norm": 0.46105241775512695, "seconds": 0.2774270449999676, "nonpadding_tokens_per_second": 243689.29135949197, "ranks": [{"compute_seconds": 0.2735766790001435, "nonpadding_tokens": 11273, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2747449299999971, "nonpadding_tokens": 19472, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2722069129999909, "nonpadding_tokens": 8390, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2750862010000219, "nonpadding_tokens": 5781, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.270790813000076, "nonpadding_tokens": 2129, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2725330109999504, "nonpadding_tokens": 3867, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27070566000020335, "nonpadding_tokens": 12195, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27127872899995964, "nonpadding_tokens": 4499, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034662899997783825, "prefetch": {"data_wait_seconds": 0.026779251999371922, "consumed_batches": 2858, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3408, "tokens": 244993802, "loss_tokens": 39418, "loss": 2.4794898586939977, "lr": 0.0002986424472227475, "optimizer_lrs": [0.019909496481516503, 0.0002986424472227475], "gradient_norm": 0.43939974904060364, "seconds": 0.22148184200000287, "nonpadding_tokens_per_second": 177973.95779288982, "ranks": [{"compute_seconds": 0.21780787799980317, "nonpadding_tokens": 6746, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2177727309999682, "nonpadding_tokens": 4031, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2183500119999735, "nonpadding_tokens": 10352, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2153092099999867, "nonpadding_tokens": 2470, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21337029500000426, "nonpadding_tokens": 4646, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2167777899999237, "nonpadding_tokens": 4389, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21912291600006029, "nonpadding_tokens": 3835, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21651159900011407, "nonpadding_tokens": 2949, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003827339999133983, "prefetch": {"data_wait_seconds": 0.026787503999415208, "consumed_batches": 2859, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3409, "tokens": 245043737, "loss_tokens": 49935, "loss": 2.3637747822168818, "lr": 0.00029864184557938515, "optimizer_lrs": [0.01990945637195901, 0.00029864184557938515], "gradient_norm": 0.4847370684146881, "seconds": 0.26835769700005585, "nonpadding_tokens_per_second": 186076.27266971816, "ranks": [{"compute_seconds": 0.26427614399995036, "nonpadding_tokens": 12236, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2630958429999737, "nonpadding_tokens": 7450, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26600450299997647, "nonpadding_tokens": 6533, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.264228581999987, "nonpadding_tokens": 4357, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26291072000003624, "nonpadding_tokens": 3808, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2654974199999742, "nonpadding_tokens": 8603, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26271434000000227, "nonpadding_tokens": 2341, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26280444799999714, "nonpadding_tokens": 4607, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039347799997813127, "prefetch": {"data_wait_seconds": 0.026798581999358362, "consumed_batches": 2860, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3410, "tokens": 245095623, "loss_tokens": 51886, "loss": 2.35756198083298, "lr": 0.00029864122028885144, "optimizer_lrs": [0.01990941468592343, 0.00029864122028885144], "gradient_norm": 0.4754661023616791, "seconds": 0.29604640899992773, "nonpadding_tokens_per_second": 175263.06154253223, "ranks": [{"compute_seconds": 0.2918547829999625, "nonpadding_tokens": 9160, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2892055900001651, "nonpadding_tokens": 4715, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28979222199996, "nonpadding_tokens": 13373, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2903991949999636, "nonpadding_tokens": 8114, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28883861700001034, "nonpadding_tokens": 2792, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2916534030000548, "nonpadding_tokens": 8149, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2925730389999899, "nonpadding_tokens": 4524, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29372161900005267, "nonpadding_tokens": 1059, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003402370000458177, "prefetch": {"data_wait_seconds": 0.026807535999481047, "consumed_batches": 2861, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3411, "tokens": 245164806, "loss_tokens": 69183, "loss": 2.4388632322969515, "lr": 0.000298640386325322, "optimizer_lrs": [0.019909359088354804, 0.000298640386325322], "gradient_norm": 0.5472530126571655, "seconds": 0.24075629200001458, "nonpadding_tokens_per_second": 287356.97590821766, "ranks": [{"compute_seconds": 0.2351234679999834, "nonpadding_tokens": 3316, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23779640699990523, "nonpadding_tokens": 13639, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2342946020000909, "nonpadding_tokens": 19927, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23724059299979672, "nonpadding_tokens": 2524, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2335607899999559, "nonpadding_tokens": 9956, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23566392399993674, "nonpadding_tokens": 3219, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2332464250000612, "nonpadding_tokens": 9805, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23839963199998238, "nonpadding_tokens": 6797, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003412989999560523, "prefetch": {"data_wait_seconds": 0.026815779999608935, "consumed_batches": 2862, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3412, "tokens": 245217016, "loss_tokens": 52210, "loss": 2.41484434854434, "lr": 0.0002986397567935614, "optimizer_lrs": [0.019909317119570762, 0.0002986397567935614], "gradient_norm": 0.4173204302787781, "seconds": 0.2559805320001942, "nonpadding_tokens_per_second": 203960.82308306315, "ranks": [{"compute_seconds": 0.2442526119998547, "nonpadding_tokens": 4320, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25045456999987437, "nonpadding_tokens": 2904, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2530903379999927, "nonpadding_tokens": 3098, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2531648970000333, "nonpadding_tokens": 3966, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2501106460001665, "nonpadding_tokens": 14572, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25249934899989057, "nonpadding_tokens": 5092, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2536803529999361, "nonpadding_tokens": 7171, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2501852020000115, "nonpadding_tokens": 11087, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003351049999764655, "prefetch": {"data_wait_seconds": 0.026823669999615163, "consumed_batches": 2863, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3413, "tokens": 245283122, "loss_tokens": 66106, "loss": 2.2809384738147824, "lr": 0.00029863895950021837, "optimizer_lrs": [0.019909263966681225, 0.00029863895950021837], "gradient_norm": 0.46945250034332275, "seconds": 0.31647711799996614, "nonpadding_tokens_per_second": 208880.82025572247, "ranks": [{"compute_seconds": 0.31288315200004035, "nonpadding_tokens": 4484, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30985640299991246, "nonpadding_tokens": 20228, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3088743150001392, "nonpadding_tokens": 10022, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30977563300007205, "nonpadding_tokens": 2768, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30821527699981743, "nonpadding_tokens": 6377, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3106314490000841, "nonpadding_tokens": 3234, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30770501899996816, "nonpadding_tokens": 10348, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3139924379997865, "nonpadding_tokens": 8645, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003556240001216793, "prefetch": {"data_wait_seconds": 0.02683298099960041, "consumed_batches": 2864, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3414, "tokens": 245353703, "loss_tokens": 70581, "loss": 1.8927617294314334, "lr": 0.000298638107978111, "optimizer_lrs": [0.019909207198540734, 0.000298638107978111], "gradient_norm": 0.4300510287284851, "seconds": 0.24405764700009058, "nonpadding_tokens_per_second": 289198.0680284679, "ranks": [{"compute_seconds": 0.2396185319998949, "nonpadding_tokens": 4406, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24133059799987677, "nonpadding_tokens": 5567, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23839115500004482, "nonpadding_tokens": 11471, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2387188420000257, "nonpadding_tokens": 4468, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24174036499994145, "nonpadding_tokens": 17173, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23896826000009241, "nonpadding_tokens": 8924, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23714536900001804, "nonpadding_tokens": 7322, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23753209500000594, "nonpadding_tokens": 11250, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003397359998871252, "prefetch": {"data_wait_seconds": 0.026841268999717727, "consumed_batches": 2865, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3415, "tokens": 245404982, "loss_tokens": 51279, "loss": 2.321080358187562, "lr": 0.0002986374891581691, "optimizer_lrs": [0.01990916594387794, 0.0002986374891581691], "gradient_norm": 0.45108622312545776, "seconds": 0.29328092900004776, "nonpadding_tokens_per_second": 174846.0091654703, "ranks": [{"compute_seconds": 0.29040341300014916, "nonpadding_tokens": 1552, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29059242700009236, "nonpadding_tokens": 10016, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2875526699999682, "nonpadding_tokens": 9403, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28820566499985034, "nonpadding_tokens": 6222, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.290272633000086, "nonpadding_tokens": 6322, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.28887377899991407, "nonpadding_tokens": 3279, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28986290700004247, "nonpadding_tokens": 6207, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29037709900012487, "nonpadding_tokens": 8278, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036438699999052915, "prefetch": {"data_wait_seconds": 0.02685057099979815, "consumed_batches": 2866, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3416, "tokens": 245466452, "loss_tokens": 61470, "loss": 2.3062662680982595, "lr": 0.0002986367471719307, "optimizer_lrs": [0.019909116478128715, 0.0002986367471719307], "gradient_norm": 0.4401223957538605, "seconds": 0.5687336680000499, "nonpadding_tokens_per_second": 108082.22452551307, "ranks": [{"compute_seconds": 0.566118404000008, "nonpadding_tokens": 9559, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.5662283419999312, "nonpadding_tokens": 5385, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.5636675010000545, "nonpadding_tokens": 11046, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.5634998840000662, "nonpadding_tokens": 8122, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.5623789159999433, "nonpadding_tokens": 5631, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.5635024670000348, "nonpadding_tokens": 13569, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.5661199290000241, "nonpadding_tokens": 3552, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.5632944419999149, "nonpadding_tokens": 4606, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004097369999271905, "prefetch": {"data_wait_seconds": 0.026859400999867944, "consumed_batches": 2867, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3417, "tokens": 245541689, "loss_tokens": 75237, "loss": 2.253583675585151, "lr": 0.00029863583873479494, "optimizer_lrs": [0.019909055915652996, 0.00029863583873479494], "gradient_norm": 0.4225386381149292, "seconds": 0.32149732700008826, "nonpadding_tokens_per_second": 234020.6082023797, "ranks": [{"compute_seconds": 0.3187913430001572, "nonpadding_tokens": 11476, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31897540399995705, "nonpadding_tokens": 13313, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31649637699979394, "nonpadding_tokens": 8697, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31907192700009546, "nonpadding_tokens": 693, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31512872099983724, "nonpadding_tokens": 10501, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3182905699998173, "nonpadding_tokens": 19649, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3156380340001306, "nonpadding_tokens": 1327, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3155803370000285, "nonpadding_tokens": 9581, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004015380000055302, "prefetch": {"data_wait_seconds": 0.026877500999901827, "consumed_batches": 2868, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3418, "tokens": 245579724, "loss_tokens": 38035, "loss": 2.1895589588536875, "lr": 0.0002986353793727024, "optimizer_lrs": [0.019909025291513494, 0.0002986353793727024], "gradient_norm": 0.4402720630168915, "seconds": 0.21563484300008895, "nonpadding_tokens_per_second": 176386.1511007491, "ranks": [{"compute_seconds": 0.21171434199982286, "nonpadding_tokens": 4194, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21031505799987826, "nonpadding_tokens": 6807, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21292507900011515, "nonpadding_tokens": 3538, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21330480500000704, "nonpadding_tokens": 4715, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2099103059999834, "nonpadding_tokens": 4687, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2113472629998796, "nonpadding_tokens": 3948, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20980087099997036, "nonpadding_tokens": 5002, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21050755600003868, "nonpadding_tokens": 5144, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003942910000205302, "prefetch": {"data_wait_seconds": 0.026901821999857134, "consumed_batches": 2869, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3419, "tokens": 245655291, "loss_tokens": 75567, "loss": 2.2356267864941044, "lr": 0.0002986344664951548, "optimizer_lrs": [0.01990896443301032, 0.0002986344664951548], "gradient_norm": 0.38569721579551697, "seconds": 0.29925095099997634, "nonpadding_tokens_per_second": 252520.50076193735, "ranks": [{"compute_seconds": 0.29697224400001687, "nonpadding_tokens": 9154, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2962697240000125, "nonpadding_tokens": 4332, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29343620899999223, "nonpadding_tokens": 16360, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29391061299998, "nonpadding_tokens": 9487, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29191536900020765, "nonpadding_tokens": 5028, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2944235159998243, "nonpadding_tokens": 17877, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29229273800001465, "nonpadding_tokens": 5219, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29686474400000407, "nonpadding_tokens": 8110, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035713000011128315, "prefetch": {"data_wait_seconds": 0.026910403999821142, "consumed_batches": 2870, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3420, "tokens": 245731795, "loss_tokens": 76504, "loss": 2.0999467756587893, "lr": 0.00029863354198894057, "optimizer_lrs": [0.019908902799262707, 0.00029863354198894057], "gradient_norm": 0.46297451853752136, "seconds": 0.26500292199989417, "nonpadding_tokens_per_second": 288691.15639423236, "ranks": [{"compute_seconds": 0.26260382400005255, "nonpadding_tokens": 3856, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2627532799999699, "nonpadding_tokens": 2506, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25949812299995756, "nonpadding_tokens": 2369, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2623676719999821, "nonpadding_tokens": 9772, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2583568000000014, "nonpadding_tokens": 16048, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2607678560000295, "nonpadding_tokens": 12063, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2585497860000032, "nonpadding_tokens": 13409, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25865581600010046, "nonpadding_tokens": 16481, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003451649999988149, "prefetch": {"data_wait_seconds": 0.026918913999864458, "consumed_batches": 2871, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3421, "tokens": 245798297, "loss_tokens": 66502, "loss": 2.2920095448257194, "lr": 0.0002986327380981814, "optimizer_lrs": [0.019908849206545426, 0.0002986327380981814], "gradient_norm": 0.44176048040390015, "seconds": 0.28272765599990635, "nonpadding_tokens_per_second": 235215.75830566086, "ranks": [{"compute_seconds": 0.2803307159999804, "nonpadding_tokens": 5490, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27999591499997223, "nonpadding_tokens": 20649, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27763212200011367, "nonpadding_tokens": 4220, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2802386739999747, "nonpadding_tokens": 14574, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2763258399997994, "nonpadding_tokens": 2546, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2790909120001288, "nonpadding_tokens": 10267, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2766507740000179, "nonpadding_tokens": 3817, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27746425199984515, "nonpadding_tokens": 4939, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038442200002464233, "prefetch": {"data_wait_seconds": 0.026930788999791844, "consumed_batches": 2872, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3422, "tokens": 245842275, "loss_tokens": 43978, "loss": 2.2938376148301423, "lr": 0.00029863220635327915, "optimizer_lrs": [0.019908813756885278, 0.00029863220635327915], "gradient_norm": 0.4597209393978119, "seconds": 0.3338178540000172, "nonpadding_tokens_per_second": 131742.50410224535, "ranks": [{"compute_seconds": 0.3289227810000739, "nonpadding_tokens": 7854, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.33147876900011397, "nonpadding_tokens": 9938, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3286431200001516, "nonpadding_tokens": 4619, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.331471110999928, "nonpadding_tokens": 2076, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3275091479999901, "nonpadding_tokens": 5708, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3308234700000412, "nonpadding_tokens": 3603, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32811711299996205, "nonpadding_tokens": 6536, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32869556899981944, "nonpadding_tokens": 3644, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000390573000004224, "prefetch": {"data_wait_seconds": 0.026942516999682198, "consumed_batches": 2873, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3423, "tokens": 245914705, "loss_tokens": 72430, "loss": 2.3054498740162916, "lr": 0.0002986313303665846, "optimizer_lrs": [0.01990875535777231, 0.0002986313303665846], "gradient_norm": 0.6136108636856079, "seconds": 0.3212812499998563, "nonpadding_tokens_per_second": 225441.1049509811, "ranks": [{"compute_seconds": 0.31750688000010996, "nonpadding_tokens": 11746, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3182643600000574, "nonpadding_tokens": 11293, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31746412400002555, "nonpadding_tokens": 11991, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31569814399995266, "nonpadding_tokens": 10684, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31433653599992795, "nonpadding_tokens": 11185, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3157570040000337, "nonpadding_tokens": 6207, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31453466299990396, "nonpadding_tokens": 4185, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3190169460001471, "nonpadding_tokens": 5139, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033556100015630363, "prefetch": {"data_wait_seconds": 0.026950383999476912, "consumed_batches": 2874, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3424, "tokens": 245968051, "loss_tokens": 53346, "loss": 2.192120484666142, "lr": 0.0002986306850081911, "optimizer_lrs": [0.01990871233387941, 0.0002986306850081911], "gradient_norm": 0.4498072564601898, "seconds": 0.2937586689999989, "nonpadding_tokens_per_second": 181598.0450265459, "ranks": [{"compute_seconds": 0.2911644769999384, "nonpadding_tokens": 11161, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2911252409999179, "nonpadding_tokens": 7022, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2876995879998958, "nonpadding_tokens": 8278, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29144593000000896, "nonpadding_tokens": 1921, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28712258299992754, "nonpadding_tokens": 6955, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29026173900001595, "nonpadding_tokens": 2825, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28675626800009013, "nonpadding_tokens": 3871, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2878654609999103, "nonpadding_tokens": 11313, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000372055000070759, "prefetch": {"data_wait_seconds": 0.026960413999361776, "consumed_batches": 2875, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3425, "tokens": 246053141, "loss_tokens": 85090, "loss": 2.481262854036902, "lr": 0.00029862965531047154, "optimizer_lrs": [0.01990864368736477, 0.00029862965531047154], "gradient_norm": 0.4056524932384491, "seconds": 0.3219690010000704, "nonpadding_tokens_per_second": 264280.10067957256, "ranks": [{"compute_seconds": 0.3171816520000448, "nonpadding_tokens": 3913, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3189492359999804, "nonpadding_tokens": 10221, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.316495046, "nonpadding_tokens": 15706, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31728530800000954, "nonpadding_tokens": 12085, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31961520799995924, "nonpadding_tokens": 8835, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3180631039999753, "nonpadding_tokens": 11600, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3153765040001417, "nonpadding_tokens": 19515, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.31640519100005804, "nonpadding_tokens": 3215, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003619379999690864, "prefetch": {"data_wait_seconds": 0.026968339999442037, "consumed_batches": 2876, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3426, "tokens": 246134463, "loss_tokens": 81322, "loss": 2.359651485760311, "lr": 0.00029862867085052867, "optimizer_lrs": [0.019908578056701914, 0.00029862867085052867], "gradient_norm": 0.40787309408187866, "seconds": 0.23008066100010183, "nonpadding_tokens_per_second": 353449.958143001, "ranks": [{"compute_seconds": 0.22621681999999055, "nonpadding_tokens": 3879, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22703214900002422, "nonpadding_tokens": 3246, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22415626600013638, "nonpadding_tokens": 6140, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2250945870000578, "nonpadding_tokens": 10916, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2274147319999429, "nonpadding_tokens": 11275, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22567860700019082, "nonpadding_tokens": 9013, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22322499899996728, "nonpadding_tokens": 11114, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22778320400016128, "nonpadding_tokens": 25739, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003821769998921809, "prefetch": {"data_wait_seconds": 0.026976350999532883, "consumed_batches": 2877, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3427, "tokens": 246214206, "loss_tokens": 79743, "loss": 2.3074533187866018, "lr": 0.00029862770516397933, "optimizer_lrs": [0.019908513677598622, 0.00029862770516397933], "gradient_norm": 0.40850841999053955, "seconds": 0.27037628499988386, "nonpadding_tokens_per_second": 294933.411042445, "ranks": [{"compute_seconds": 0.2681349490001139, "nonpadding_tokens": 10893, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2676703380000163, "nonpadding_tokens": 10212, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2667535850000604, "nonpadding_tokens": 19809, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26515458399990166, "nonpadding_tokens": 12197, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2637449219998871, "nonpadding_tokens": 2265, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.266547884999909, "nonpadding_tokens": 4775, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26309936200004813, "nonpadding_tokens": 8819, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2680890179999551, "nonpadding_tokens": 10773, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035464899997350585, "prefetch": {"data_wait_seconds": 0.02698580199962919, "consumed_batches": 2878, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3428, "tokens": 246291977, "loss_tokens": 77771, "loss": 2.0498454597472064, "lr": 0.0002986267630326008, "optimizer_lrs": [0.019908450868840053, 0.0002986267630326008], "gradient_norm": 0.7745329141616821, "seconds": 0.2640584129999297, "nonpadding_tokens_per_second": 294521.9548828414, "ranks": [{"compute_seconds": 0.2617691629998262, "nonpadding_tokens": 10570, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25838531099998363, "nonpadding_tokens": 7496, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2585208809998676, "nonpadding_tokens": 2356, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2588589009999396, "nonpadding_tokens": 12016, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2575375020001047, "nonpadding_tokens": 10276, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2603111739999804, "nonpadding_tokens": 24815, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2575426039998092, "nonpadding_tokens": 4651, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.25843219900002623, "nonpadding_tokens": 5591, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038519000008818693, "prefetch": {"data_wait_seconds": 0.026993786999540248, "consumed_batches": 2879, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3429, "tokens": 246402146, "loss_tokens": 110169, "loss": 2.1774323709482704, "lr": 0.00029862542787570064, "optimizer_lrs": [0.019908361858380044, 0.00029862542787570064], "gradient_norm": 0.45211657881736755, "seconds": 0.26995944100008273, "nonpadding_tokens_per_second": 408094.63670494943, "ranks": [{"compute_seconds": 0.2655209649999506, "nonpadding_tokens": 3510, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2676268269999582, "nonpadding_tokens": 21366, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26430792099995415, "nonpadding_tokens": 4980, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.26550000399993223, "nonpadding_tokens": 19058, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.263851184999794, "nonpadding_tokens": 9080, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26666614499981733, "nonpadding_tokens": 20998, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2636687780000102, "nonpadding_tokens": 21403, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.263779655999997, "nonpadding_tokens": 9774, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035425500004748756, "prefetch": {"data_wait_seconds": 0.027002383999573, "consumed_batches": 2880, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3430, "tokens": 246443162, "loss_tokens": 41016, "loss": 2.1440428735127757, "lr": 0.00029862493063092334, "optimizer_lrs": [0.019908328708728225, 0.00029862493063092334], "gradient_norm": 0.5622600317001343, "seconds": 0.26697043399985887, "nonpadding_tokens_per_second": 153634.9901578303, "ranks": [{"compute_seconds": 0.2627014100000906, "nonpadding_tokens": 7620, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2613345469999331, "nonpadding_tokens": 1554, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2637139680000473, "nonpadding_tokens": 2843, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2611991129999751, "nonpadding_tokens": 4103, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2646253749999232, "nonpadding_tokens": 3070, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2626461739998831, "nonpadding_tokens": 6350, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25975883400019484, "nonpadding_tokens": 2863, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2604530449998492, "nonpadding_tokens": 12613, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036472400006459793, "prefetch": {"data_wait_seconds": 0.027011519999362577, "consumed_batches": 2881, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3431, "tokens": 246493042, "loss_tokens": 49880, "loss": 2.3434733986567764, "lr": 0.00029862432580563556, "optimizer_lrs": [0.01990828838704237, 0.00029862432580563556], "gradient_norm": 0.43645384907722473, "seconds": 0.2518915160001143, "nonpadding_tokens_per_second": 198021.75473022828, "ranks": [{"compute_seconds": 0.24043956700006675, "nonpadding_tokens": 5695, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24192089100006342, "nonpadding_tokens": 2438, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24242245899995396, "nonpadding_tokens": 13777, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24534940900002766, "nonpadding_tokens": 2011, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24566729200000736, "nonpadding_tokens": 7208, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24425522100000308, "nonpadding_tokens": 8305, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24521416900006443, "nonpadding_tokens": 1907, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24188463199993748, "nonpadding_tokens": 8539, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004361559999779274, "prefetch": {"data_wait_seconds": 0.027019542999326518, "consumed_batches": 2882, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3432, "tokens": 246548601, "loss_tokens": 55559, "loss": 2.570999129303983, "lr": 0.0002986236519632658, "optimizer_lrs": [0.019908243464217722, 0.0002986236519632658], "gradient_norm": 0.4115248918533325, "seconds": 0.31925527599992165, "nonpadding_tokens_per_second": 174026.88123473217, "ranks": [{"compute_seconds": 0.3165876949999529, "nonpadding_tokens": 17017, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3135819219999121, "nonpadding_tokens": 3541, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31607846899987635, "nonpadding_tokens": 3965, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31644188399991435, "nonpadding_tokens": 1515, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3169824229998994, "nonpadding_tokens": 11082, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31475305000003573, "nonpadding_tokens": 10879, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31260933400017166, "nonpadding_tokens": 4703, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3123855630001344, "nonpadding_tokens": 2857, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00032863900014490355, "prefetch": {"data_wait_seconds": 0.02702718799942261, "consumed_batches": 2883, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3433, "tokens": 246604380, "loss_tokens": 55779, "loss": 2.193726161503433, "lr": 0.00029862297528752236, "optimizer_lrs": [0.019908198352501492, 0.00029862297528752236], "gradient_norm": 0.6115458011627197, "seconds": 0.29299282399983895, "nonpadding_tokens_per_second": 190376.67625617568, "ranks": [{"compute_seconds": 0.2885037379999176, "nonpadding_tokens": 14720, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2907030000001214, "nonpadding_tokens": 11231, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28960536700014927, "nonpadding_tokens": 13539, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28809939000007034, "nonpadding_tokens": 3358, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28723989199988864, "nonpadding_tokens": 4781, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2894098199999462, "nonpadding_tokens": 2138, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2863637779998953, "nonpadding_tokens": 3135, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.28754789799995706, "nonpadding_tokens": 2877, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038484900005641975, "prefetch": {"data_wait_seconds": 0.027035985999376067, "consumed_batches": 2884, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3434, "tokens": 246676369, "loss_tokens": 71989, "loss": 2.193367649918738, "lr": 0.00029862210171769776, "optimizer_lrs": [0.019908140114513185, 0.00029862210171769776], "gradient_norm": 0.4364081919193268, "seconds": 0.28560469500007457, "nonpadding_tokens_per_second": 252058.18132640014, "ranks": [{"compute_seconds": 0.28111208799987253, "nonpadding_tokens": 5080, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2800077109998256, "nonpadding_tokens": 21930, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28068989799999144, "nonpadding_tokens": 5434, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28118409900002916, "nonpadding_tokens": 7948, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27930523300005916, "nonpadding_tokens": 10878, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2819470139997975, "nonpadding_tokens": 11914, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28331653500003995, "nonpadding_tokens": 4407, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.279758873999981, "nonpadding_tokens": 4398, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003524229998674855, "prefetch": {"data_wait_seconds": 0.027044865999371837, "consumed_batches": 2885, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3435, "tokens": 246721981, "loss_tokens": 45612, "loss": 2.391387717322196, "lr": 0.0002986215480840329, "optimizer_lrs": [0.019908103205602195, 0.0002986215480840329], "gradient_norm": 0.42638102173805237, "seconds": 0.22643606399992677, "nonpadding_tokens_per_second": 201434.34395686522, "ranks": [{"compute_seconds": 0.2242074479997882, "nonpadding_tokens": 4782, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22261887200011188, "nonpadding_tokens": 7487, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22268282300001374, "nonpadding_tokens": 2457, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22292973000003258, "nonpadding_tokens": 11569, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22371438899995155, "nonpadding_tokens": 5059, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22098439499995948, "nonpadding_tokens": 4229, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.21879659100000026, "nonpadding_tokens": 7151, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2189123020000352, "nonpadding_tokens": 2878, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035940399993705796, "prefetch": {"data_wait_seconds": 0.027054330999320086, "consumed_batches": 2886, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3436, "tokens": 246790149, "loss_tokens": 68168, "loss": 2.056170738102922, "lr": 0.00029862072046178803, "optimizer_lrs": [0.01990804803078587, 0.00029862072046178803], "gradient_norm": 0.8226335048675537, "seconds": 0.2622578849998263, "nonpadding_tokens_per_second": 259927.3611927632, "ranks": [{"compute_seconds": 0.2578771619998861, "nonpadding_tokens": 3062, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2569998819999455, "nonpadding_tokens": 6867, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2591952399998263, "nonpadding_tokens": 5265, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.258000906999996, "nonpadding_tokens": 20796, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.26002322399995137, "nonpadding_tokens": 13161, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25907616800009237, "nonpadding_tokens": 8672, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2598380780000298, "nonpadding_tokens": 7039, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2569353620001493, "nonpadding_tokens": 3306, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035403900005803735, "prefetch": {"data_wait_seconds": 0.02706370799951685, "consumed_batches": 2887, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3437, "tokens": 246846576, "loss_tokens": 56427, "loss": 2.1894566209438744, "lr": 0.00029862003519915943, "optimizer_lrs": [0.01990800234661063, 0.00029862003519915943], "gradient_norm": 0.42106395959854126, "seconds": 0.25964638499999637, "nonpadding_tokens_per_second": 217322.49420688368, "ranks": [{"compute_seconds": 0.2573592149999513, "nonpadding_tokens": 7485, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25700371199991423, "nonpadding_tokens": 17486, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25362098399978095, "nonpadding_tokens": 10686, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25727992000020095, "nonpadding_tokens": 4160, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25313322900001367, "nonpadding_tokens": 5982, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25548479600001883, "nonpadding_tokens": 4014, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2571469320000688, "nonpadding_tokens": 3218, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2536223249999239, "nonpadding_tokens": 3396, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003470519998245436, "prefetch": {"data_wait_seconds": 0.027072926999338165, "consumed_batches": 2888, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3438, "tokens": 246922546, "loss_tokens": 75970, "loss": 2.5298287975516653, "lr": 0.0002986191123343622, "optimizer_lrs": [0.019907940822290814, 0.0002986191123343622], "gradient_norm": 0.45311951637268066, "seconds": 0.2925836590000017, "nonpadding_tokens_per_second": 259652.23163744615, "ranks": [{"compute_seconds": 0.2884514659999695, "nonpadding_tokens": 9033, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2858680580000055, "nonpadding_tokens": 2370, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2886347270000442, "nonpadding_tokens": 7356, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2868105289999221, "nonpadding_tokens": 5330, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28555274799987274, "nonpadding_tokens": 6594, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2879632449999008, "nonpadding_tokens": 17629, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28537265399995704, "nonpadding_tokens": 9146, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2902510579999671, "nonpadding_tokens": 18512, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003999230000317766, "prefetch": {"data_wait_seconds": 0.027082447999376313, "consumed_batches": 2889, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3439, "tokens": 246968904, "loss_tokens": 46358, "loss": 2.5139901351978082, "lr": 0.0002986185490380521, "optimizer_lrs": [0.019907903269203477, 0.0002986185490380521], "gradient_norm": 0.49161824584007263, "seconds": 0.2780628489999799, "nonpadding_tokens_per_second": 166717.70488837705, "ranks": [{"compute_seconds": 0.2754887329999747, "nonpadding_tokens": 1989, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27198252700009107, "nonpadding_tokens": 7085, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2718242939999982, "nonpadding_tokens": 6991, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27230912000004537, "nonpadding_tokens": 5123, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27092502599998625, "nonpadding_tokens": 4747, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27383010099993044, "nonpadding_tokens": 4470, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2707172880000144, "nonpadding_tokens": 10450, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2720035840000037, "nonpadding_tokens": 5503, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003553220001322188, "prefetch": {"data_wait_seconds": 0.027092087999292858, "consumed_batches": 2890, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3440, "tokens": 247018838, "loss_tokens": 49934, "loss": 2.147639102365122, "lr": 0.0002986179421619073, "optimizer_lrs": [0.01990786281079382, 0.0002986179421619073], "gradient_norm": 0.4181794822216034, "seconds": 0.27907653400006893, "nonpadding_tokens_per_second": 178925.8282818851, "ranks": [{"compute_seconds": 0.27486049899994214, "nonpadding_tokens": 5374, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2728977120000309, "nonpadding_tokens": 4044, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2735857019999912, "nonpadding_tokens": 9572, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27385555199998635, "nonpadding_tokens": 3496, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27676000299993575, "nonpadding_tokens": 8770, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27478646800000206, "nonpadding_tokens": 2615, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2762340519998361, "nonpadding_tokens": 13044, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27301808700008223, "nonpadding_tokens": 3019, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037965700016684423, "prefetch": {"data_wait_seconds": 0.027101015999235756, "consumed_batches": 2891, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3441, "tokens": 247063999, "loss_tokens": 45161, "loss": 2.235404995460685, "lr": 0.00029861739318054925, "optimizer_lrs": [0.019907826212036617, 0.00029861739318054925], "gradient_norm": 0.4394579231739044, "seconds": 0.2411544750000303, "nonpadding_tokens_per_second": 187270.0060821775, "ranks": [{"compute_seconds": 0.23561812000002647, "nonpadding_tokens": 5538, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23854956699983632, "nonpadding_tokens": 4782, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2377683380000235, "nonpadding_tokens": 8588, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23627242800012027, "nonpadding_tokens": 2787, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23885471400012648, "nonpadding_tokens": 4773, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23723117800000182, "nonpadding_tokens": 11037, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23835298800008786, "nonpadding_tokens": 5619, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2388426420000087, "nonpadding_tokens": 2037, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000333049999881041, "prefetch": {"data_wait_seconds": 0.02710921799912569, "consumed_batches": 2892, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3442, "tokens": 247122290, "loss_tokens": 58291, "loss": 2.2196832218953184, "lr": 0.000298616684429348, "optimizer_lrs": [0.019907778961956536, 0.000298616684429348], "gradient_norm": 0.46720409393310547, "seconds": 0.3347758770000837, "nonpadding_tokens_per_second": 174119.47516154344, "ranks": [{"compute_seconds": 0.33219066499987093, "nonpadding_tokens": 5689, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.327268783999898, "nonpadding_tokens": 9079, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.33061491600005866, "nonpadding_tokens": 4188, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.32781759199997396, "nonpadding_tokens": 6510, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3314540180001586, "nonpadding_tokens": 7562, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32908554699997694, "nonpadding_tokens": 6309, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.33141196899987335, "nonpadding_tokens": 11906, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.33169582499999706, "nonpadding_tokens": 7048, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00047031900021465844, "prefetch": {"data_wait_seconds": 0.027119209999185045, "consumed_batches": 2893, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3443, "tokens": 247186705, "loss_tokens": 64415, "loss": 2.348627309244741, "lr": 0.00029861590100723585, "optimizer_lrs": [0.019907726733815724, 0.00029861590100723585], "gradient_norm": 0.45622119307518005, "seconds": 0.24378770700013774, "nonpadding_tokens_per_second": 264225.7921559745, "ranks": [{"compute_seconds": 0.24148770600004354, "nonpadding_tokens": 6222, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23757583899987367, "nonpadding_tokens": 1887, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2402306999999837, "nonpadding_tokens": 14953, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23887367800011816, "nonpadding_tokens": 12092, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23687343000005967, "nonpadding_tokens": 7660, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2386436080000749, "nonpadding_tokens": 3855, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2372290560001602, "nonpadding_tokens": 3465, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23787604900007864, "nonpadding_tokens": 14281, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003982719999839901, "prefetch": {"data_wait_seconds": 0.02712916999935544, "consumed_batches": 2894, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3444, "tokens": 247244778, "loss_tokens": 58073, "loss": 2.0880140889483925, "lr": 0.0002986151945280854, "optimizer_lrs": [0.019907679635205695, 0.0002986151945280854], "gradient_norm": 0.5221054553985596, "seconds": 0.2226583980000214, "nonpadding_tokens_per_second": 260816.57158062558, "ranks": [{"compute_seconds": 0.2179692110000815, "nonpadding_tokens": 3527, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21489313699999002, "nonpadding_tokens": 4389, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21772566999993614, "nonpadding_tokens": 6297, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2185483219998332, "nonpadding_tokens": 5993, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21761941599993406, "nonpadding_tokens": 12615, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22012687799997366, "nonpadding_tokens": 2548, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2173279779999575, "nonpadding_tokens": 13833, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.21808005399998365, "nonpadding_tokens": 8871, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003663059999325924, "prefetch": {"data_wait_seconds": 0.027138421999282514, "consumed_batches": 2895, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3445, "tokens": 247323853, "loss_tokens": 79075, "loss": 2.532198071451154, "lr": 0.00029861423226363855, "optimizer_lrs": [0.01990761548424257, 0.00029861423226363855], "gradient_norm": 0.7406593561172485, "seconds": 0.30267180600003485, "nonpadding_tokens_per_second": 261256.57703311453, "ranks": [{"compute_seconds": 0.2996240940001371, "nonpadding_tokens": 13101, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2986818019999191, "nonpadding_tokens": 18285, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2972881860000598, "nonpadding_tokens": 3773, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30028948999984095, "nonpadding_tokens": 10584, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2967036499999267, "nonpadding_tokens": 9476, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2991347180000048, "nonpadding_tokens": 4707, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3003844630000003, "nonpadding_tokens": 2560, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29701960499983215, "nonpadding_tokens": 16589, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003678400000808324, "prefetch": {"data_wait_seconds": 0.027146674999315223, "consumed_batches": 2896, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3446, "tokens": 247413152, "loss_tokens": 89299, "loss": 2.13187262735305, "lr": 0.00029861314518348896, "optimizer_lrs": [0.0199075430122326, 0.00029861314518348896], "gradient_norm": 0.41537418961524963, "seconds": 0.256972499000085, "nonpadding_tokens_per_second": 347504.1117141895, "ranks": [{"compute_seconds": 0.25204767599984734, "nonpadding_tokens": 4228, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25455921999991915, "nonpadding_tokens": 16091, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25386075900019023, "nonpadding_tokens": 10626, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25179285599983814, "nonpadding_tokens": 18603, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25462975400000687, "nonpadding_tokens": 14745, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2530652339999051, "nonpadding_tokens": 8296, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.25441187899991746, "nonpadding_tokens": 7601, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2508029910000005, "nonpadding_tokens": 9109, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003861390000565734, "prefetch": {"data_wait_seconds": 0.027155880999274018, "consumed_batches": 2897, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3447, "tokens": 247461353, "loss_tokens": 48201, "loss": 2.0625372787908964, "lr": 0.00029861255823307215, "optimizer_lrs": [0.019907503882204813, 0.00029861255823307215], "gradient_norm": 0.36563414335250854, "seconds": 0.2491069359998619, "nonpadding_tokens_per_second": 193495.21444086457, "ranks": [{"compute_seconds": 0.24610249300008036, "nonpadding_tokens": 5872, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2457996020000337, "nonpadding_tokens": 4616, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24319294599990826, "nonpadding_tokens": 3336, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2433035030001065, "nonpadding_tokens": 12267, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2420986649999577, "nonpadding_tokens": 9449, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24408662100017864, "nonpadding_tokens": 4025, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2468394749998879, "nonpadding_tokens": 6033, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24287971900002958, "nonpadding_tokens": 2603, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003553089998149517, "prefetch": {"data_wait_seconds": 0.027164579999137572, "consumed_batches": 2898, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3448, "tokens": 247493169, "loss_tokens": 31816, "loss": 2.2091594539697006, "lr": 0.0002986121707374413, "optimizer_lrs": [0.019907478049162753, 0.0002986121707374413], "gradient_norm": 0.42725688219070435, "seconds": 0.2513525199999549, "nonpadding_tokens_per_second": 126579.19642104924, "ranks": [{"compute_seconds": 0.24638254299998152, "nonpadding_tokens": 4257, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24906645999999455, "nonpadding_tokens": 2511, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24785648900001434, "nonpadding_tokens": 7965, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24654132199998458, "nonpadding_tokens": 1083, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2451675489999161, "nonpadding_tokens": 5591, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24754088700001375, "nonpadding_tokens": 6011, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24531712899988634, "nonpadding_tokens": 2413, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24535191199993278, "nonpadding_tokens": 1985, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039767800012668886, "prefetch": {"data_wait_seconds": 0.027173346999191494, "consumed_batches": 2899, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3449, "tokens": 247551956, "loss_tokens": 58787, "loss": 2.1480313727950056, "lr": 0.00029861145461313355, "optimizer_lrs": [0.01990743030754224, 0.00029861145461313355], "gradient_norm": 0.6057949066162109, "seconds": 0.20521088500004225, "nonpadding_tokens_per_second": 286471.1586814115, "ranks": [{"compute_seconds": 0.20298607600011565, "nonpadding_tokens": 5463, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.19954188400015482, "nonpadding_tokens": 5032, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2000349699999333, "nonpadding_tokens": 5070, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20003208399998584, "nonpadding_tokens": 11175, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.20290849999992133, "nonpadding_tokens": 5817, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20036843999992016, "nonpadding_tokens": 6011, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19889524199993502, "nonpadding_tokens": 7219, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.19940556799997466, "nonpadding_tokens": 13000, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034159699998781434, "prefetch": {"data_wait_seconds": 0.027182327999256017, "consumed_batches": 2900, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3450, "tokens": 247595769, "loss_tokens": 43813, "loss": 2.142103585123137, "lr": 0.00029861092077776273, "optimizer_lrs": [0.019907394718517518, 0.00029861092077776273], "gradient_norm": 0.3880142271518707, "seconds": 0.39470615300001555, "nonpadding_tokens_per_second": 111001.56323126353, "ranks": [{"compute_seconds": 0.39116473399985807, "nonpadding_tokens": 1850, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.39114961100017354, "nonpadding_tokens": 10770, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.39087324299998727, "nonpadding_tokens": 5297, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3915316200000234, "nonpadding_tokens": 7871, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3919527600000947, "nonpadding_tokens": 2881, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.38963859099999354, "nonpadding_tokens": 6048, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3871894090000296, "nonpadding_tokens": 3858, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3922678710000582, "nonpadding_tokens": 5238, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039142900004662806, "prefetch": {"data_wait_seconds": 0.027192255999310078, "consumed_batches": 2901, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3451, "tokens": 247640715, "loss_tokens": 44946, "loss": 2.246693254127175, "lr": 0.00029861037303139894, "optimizer_lrs": [0.019907358202093266, 0.00029861037303139894], "gradient_norm": 0.48688003420829773, "seconds": 0.21171567199985475, "nonpadding_tokens_per_second": 212294.1564761953, "ranks": [{"compute_seconds": 0.20897370800003046, "nonpadding_tokens": 3529, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20884522699998342, "nonpadding_tokens": 2660, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.20809000299982472, "nonpadding_tokens": 4538, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2085580509999545, "nonpadding_tokens": 4741, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.20930514800011224, "nonpadding_tokens": 1652, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2076006789998246, "nonpadding_tokens": 10951, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20892993400002524, "nonpadding_tokens": 7581, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.20573409400003584, "nonpadding_tokens": 9294, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00043298599985064357, "prefetch": {"data_wait_seconds": 0.0272035129994066, "consumed_batches": 2902, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3452, "tokens": 247698911, "loss_tokens": 58196, "loss": 2.3439204906694617, "lr": 0.00029860966365081136, "optimizer_lrs": [0.019907310910054092, 0.00029860966365081136], "gradient_norm": 0.5723179578781128, "seconds": 0.20530006199987838, "nonpadding_tokens_per_second": 283468.00986370124, "ranks": [{"compute_seconds": 0.2023457520001557, "nonpadding_tokens": 6757, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20258774399985668, "nonpadding_tokens": 3605, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.201493414999959, "nonpadding_tokens": 11688, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.20229378399994857, "nonpadding_tokens": 5596, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.202965445000018, "nonpadding_tokens": 11659, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.20124374900001385, "nonpadding_tokens": 11823, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.20221217899984367, "nonpadding_tokens": 4952, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.19877151200012122, "nonpadding_tokens": 2116, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003483819998564286, "prefetch": {"data_wait_seconds": 0.027212491999492272, "consumed_batches": 2903, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3453, "tokens": 247748492, "loss_tokens": 49581, "loss": 2.573882888606523, "lr": 0.00029860905914076615, "optimizer_lrs": [0.019907270609384412, 0.00029860905914076615], "gradient_norm": 0.5127651691436768, "seconds": 0.2861732579999625, "nonpadding_tokens_per_second": 173255.1823553216, "ranks": [{"compute_seconds": 0.2814537399999608, "nonpadding_tokens": 18201, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2794055209999442, "nonpadding_tokens": 13891, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2796234300001288, "nonpadding_tokens": 1729, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.28240809299995817, "nonpadding_tokens": 3708, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2790512970000236, "nonpadding_tokens": 3936, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2813818600000104, "nonpadding_tokens": 2755, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27872386200010624, "nonpadding_tokens": 1179, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2838082610001038, "nonpadding_tokens": 4182, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00038060699989728164, "prefetch": {"data_wait_seconds": 0.027221414999488047, "consumed_batches": 2904, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3454, "tokens": 247843262, "loss_tokens": 94770, "loss": 2.1127545636804896, "lr": 0.00029860790330588305, "optimizer_lrs": [0.019907193553725538, 0.00029860790330588305], "gradient_norm": 0.47966617345809937, "seconds": 0.30099773200004165, "nonpadding_tokens_per_second": 314852.8707185969, "ranks": [{"compute_seconds": 0.29639939799994863, "nonpadding_tokens": 8881, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2946316160000606, "nonpadding_tokens": 6287, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29718962200013266, "nonpadding_tokens": 8892, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29791011799989064, "nonpadding_tokens": 11961, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2986454169999888, "nonpadding_tokens": 13610, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2967812430001686, "nonpadding_tokens": 17385, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2940093669999442, "nonpadding_tokens": 9904, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29471134600021287, "nonpadding_tokens": 17850, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040927100008048, "prefetch": {"data_wait_seconds": 0.027230250999494388, "consumed_batches": 2905, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3455, "tokens": 247931768, "loss_tokens": 88506, "loss": 1.8348441772309223, "lr": 0.000298606823436821, "optimizer_lrs": [0.019907121562454735, 0.000298606823436821], "gradient_norm": 0.5720623135566711, "seconds": 0.3132356799999343, "nonpadding_tokens_per_second": 282554.0181119168, "ranks": [{"compute_seconds": 0.3104818439999235, "nonpadding_tokens": 17964, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3106857459999901, "nonpadding_tokens": 14329, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3082263020000937, "nonpadding_tokens": 25814, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3082133409998278, "nonpadding_tokens": 8327, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30704745599996386, "nonpadding_tokens": 3482, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3080703339999218, "nonpadding_tokens": 4358, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.310863301000154, "nonpadding_tokens": 2515, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.307073285000115, "nonpadding_tokens": 11717, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000420662000124139, "prefetch": {"data_wait_seconds": 0.027239716999432062, "consumed_batches": 2906, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3456, "tokens": 247961920, "loss_tokens": 30152, "loss": 2.105555161141881, "lr": 0.00029860645545464685, "optimizer_lrs": [0.01990709703030979, 0.00029860645545464685], "gradient_norm": 0.4836275279521942, "seconds": 0.23369164199993975, "nonpadding_tokens_per_second": 129024.72566822811, "ranks": [{"compute_seconds": 0.23029157599989958, "nonpadding_tokens": 3655, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23088757200002874, "nonpadding_tokens": 2401, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22764940399997613, "nonpadding_tokens": 1898, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23052135100010673, "nonpadding_tokens": 1877, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22685283900000286, "nonpadding_tokens": 4643, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2297278710000228, "nonpadding_tokens": 5245, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2266881729999568, "nonpadding_tokens": 4498, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2313448200000039, "nonpadding_tokens": 5935, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003692719999435212, "prefetch": {"data_wait_seconds": 0.027247605999448865, "consumed_batches": 2907, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3457, "tokens": 248019409, "loss_tokens": 57489, "loss": 2.0228619822922647, "lr": 0.0002986057537112816, "optimizer_lrs": [0.019907050247418775, 0.0002986057537112816], "gradient_norm": 0.4248450696468353, "seconds": 0.24623835600004895, "nonpadding_tokens_per_second": 233468.90766273867, "ranks": [{"compute_seconds": 0.2418173849998766, "nonpadding_tokens": 3349, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2435286290001386, "nonpadding_tokens": 2944, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24089169500007301, "nonpadding_tokens": 11242, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24128874499979247, "nonpadding_tokens": 9337, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.24393166600020777, "nonpadding_tokens": 1827, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24113383600001725, "nonpadding_tokens": 6496, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23983667300012712, "nonpadding_tokens": 3615, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24075968200008901, "nonpadding_tokens": 18679, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00039404799986186845, "prefetch": {"data_wait_seconds": 0.027255637999545, "consumed_batches": 2908, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3458, "tokens": 248080098, "loss_tokens": 60689, "loss": 2.1242926281945658, "lr": 0.00029860501271622983, "optimizer_lrs": [0.019907000847748657, 0.00029860501271622983], "gradient_norm": 0.39005592465400696, "seconds": 0.24767128200005573, "nonpadding_tokens_per_second": 245038.50228379059, "ranks": [{"compute_seconds": 0.24268619700001182, "nonpadding_tokens": 3444, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2449410040001112, "nonpadding_tokens": 9785, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24204090000012002, "nonpadding_tokens": 6609, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.24227357300014774, "nonpadding_tokens": 4089, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2408705599998484, "nonpadding_tokens": 7792, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24407283599998664, "nonpadding_tokens": 8259, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.244681392000075, "nonpadding_tokens": 3466, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.24540077000006022, "nonpadding_tokens": 17245, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00034982500005753536, "prefetch": {"data_wait_seconds": 0.02726410899954601, "consumed_batches": 2909, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3459, "tokens": 248144797, "loss_tokens": 64699, "loss": 2.4163824981838977, "lr": 0.0002986042225445747, "optimizer_lrs": [0.019906948169638315, 0.0002986042225445747], "gradient_norm": 0.45002487301826477, "seconds": 0.2851682560001336, "nonpadding_tokens_per_second": 226880.0914501847, "ranks": [{"compute_seconds": 0.28166618999989623, "nonpadding_tokens": 2077, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2789957510001386, "nonpadding_tokens": 9969, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2814971400000559, "nonpadding_tokens": 7085, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27836741600003734, "nonpadding_tokens": 7799, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2828593459998956, "nonpadding_tokens": 6062, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2795207600001959, "nonpadding_tokens": 10489, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2823682440000539, "nonpadding_tokens": 9200, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2774095610000131, "nonpadding_tokens": 12018, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00033435299997108814, "prefetch": {"data_wait_seconds": 0.027272354999652748, "consumed_batches": 2910, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3460, "tokens": 248201255, "loss_tokens": 56458, "loss": 2.192300305094052, "lr": 0.00029860353283875845, "optimizer_lrs": [0.019906902189250564, 0.00029860353283875845], "gradient_norm": 0.4541151225566864, "seconds": 0.24269782900000791, "nonpadding_tokens_per_second": 232626.720365093, "ranks": [{"compute_seconds": 0.24044867999987218, "nonpadding_tokens": 5692, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23996251399989887, "nonpadding_tokens": 9195, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23716594699999405, "nonpadding_tokens": 5924, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23720759499997257, "nonpadding_tokens": 7776, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2399004590001823, "nonpadding_tokens": 6042, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23687891899999158, "nonpadding_tokens": 14488, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.23595011200018234, "nonpadding_tokens": 1555, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2361614760000066, "nonpadding_tokens": 5786, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003867539999191649, "prefetch": {"data_wait_seconds": 0.02728028099973301, "consumed_batches": 2911, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3461, "tokens": 248246470, "loss_tokens": 45215, "loss": 2.357891117438903, "lr": 0.0002986029803581821, "optimizer_lrs": [0.01990686535721214, 0.0002986029803581821], "gradient_norm": 0.4768058657646179, "seconds": 0.2991350400000101, "nonpadding_tokens_per_second": 151152.46946662775, "ranks": [{"compute_seconds": 0.2961617099999785, "nonpadding_tokens": 7021, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2959321620000992, "nonpadding_tokens": 6630, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29296326000007866, "nonpadding_tokens": 12571, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2958072720000473, "nonpadding_tokens": 4371, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2919271729999764, "nonpadding_tokens": 1652, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2933757620000961, "nonpadding_tokens": 5639, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2917316819998632, "nonpadding_tokens": 4769, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2968314490001376, "nonpadding_tokens": 2562, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003540560001056292, "prefetch": {"data_wait_seconds": 0.02728919199985569, "consumed_batches": 2912, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3462, "tokens": 248301316, "loss_tokens": 54846, "loss": 2.1779907035152974, "lr": 0.0002986023100508442, "optimizer_lrs": [0.01990682067005628, 0.0002986023100508442], "gradient_norm": 0.4497646391391754, "seconds": 0.2320784860000913, "nonpadding_tokens_per_second": 236325.22318323996, "ranks": [{"compute_seconds": 0.2278544050000164, "nonpadding_tokens": 5362, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22716738899998745, "nonpadding_tokens": 3481, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22977898299996014, "nonpadding_tokens": 6178, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22648651100007555, "nonpadding_tokens": 7154, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22442801599981976, "nonpadding_tokens": 1568, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22739613800013103, "nonpadding_tokens": 15723, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22475455400012834, "nonpadding_tokens": 3278, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22541723299991645, "nonpadding_tokens": 12102, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035887299986825383, "prefetch": {"data_wait_seconds": 0.02729783999980384, "consumed_batches": 2913, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3463, "tokens": 248376719, "loss_tokens": 75403, "loss": 2.217706647945042, "lr": 0.0002986013882424378, "optimizer_lrs": [0.019906759216162518, 0.0002986013882424378], "gradient_norm": 0.5001145005226135, "seconds": 0.3168289749999076, "nonpadding_tokens_per_second": 237992.7530303123, "ranks": [{"compute_seconds": 0.3133330279999882, "nonpadding_tokens": 4910, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31082848699998067, "nonpadding_tokens": 14172, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3114551339999707, "nonpadding_tokens": 1936, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3142678750000414, "nonpadding_tokens": 14081, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31054532100006327, "nonpadding_tokens": 11625, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3126611069999399, "nonpadding_tokens": 11074, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3144809890000033, "nonpadding_tokens": 12670, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3108442590000777, "nonpadding_tokens": 4935, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036860099999103113, "prefetch": {"data_wait_seconds": 0.02730760699974155, "consumed_batches": 2914, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3464, "tokens": 248434874, "loss_tokens": 58155, "loss": 2.0827895279855557, "lr": 0.0002986006770858906, "optimizer_lrs": [0.019906711805726042, 0.0002986006770858906], "gradient_norm": 0.45240259170532227, "seconds": 0.23078828200004864, "nonpadding_tokens_per_second": 251984.19736053905, "ranks": [{"compute_seconds": 0.22775585800013687, "nonpadding_tokens": 9492, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.227838969000004, "nonpadding_tokens": 10342, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22450898500005678, "nonpadding_tokens": 10273, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22582353300003888, "nonpadding_tokens": 2066, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22368120900000577, "nonpadding_tokens": 14985, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22630993000007038, "nonpadding_tokens": 2180, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22773728200013466, "nonpadding_tokens": 3079, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22845351399996616, "nonpadding_tokens": 5738, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003987689999576105, "prefetch": {"data_wait_seconds": 0.02731578199991702, "consumed_batches": 2915, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3465, "tokens": 248499735, "loss_tokens": 64861, "loss": 2.1224344174465397, "lr": 0.0002985998837119731, "optimizer_lrs": [0.019906658914131542, 0.0002985998837119731], "gradient_norm": 0.4606076180934906, "seconds": 0.28859123099982753, "nonpadding_tokens_per_second": 224750.41869875375, "ranks": [{"compute_seconds": 0.28479180999988785, "nonpadding_tokens": 3564, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28607678499997746, "nonpadding_tokens": 19658, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28326167699992766, "nonpadding_tokens": 3015, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.286185574000001, "nonpadding_tokens": 3134, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.28634111499991377, "nonpadding_tokens": 5063, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2836863480001739, "nonpadding_tokens": 10226, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2817855779999263, "nonpadding_tokens": 10565, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2828484980000212, "nonpadding_tokens": 9636, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036786999999094405, "prefetch": {"data_wait_seconds": 0.027325242999907573, "consumed_batches": 2916, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3466, "tokens": 248561817, "loss_tokens": 62082, "loss": 2.507141764118424, "lr": 0.0002985991241210299, "optimizer_lrs": [0.019906608274735327, 0.0002985991241210299], "gradient_norm": 0.5171188712120056, "seconds": 0.23751959200012607, "nonpadding_tokens_per_second": 261376.33311515223, "ranks": [{"compute_seconds": 0.23368627900003958, "nonpadding_tokens": 15123, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2348815000000286, "nonpadding_tokens": 3291, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23420908399998552, "nonpadding_tokens": 2192, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23292905300013445, "nonpadding_tokens": 10482, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23496967000005498, "nonpadding_tokens": 8134, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.23364865899998222, "nonpadding_tokens": 4951, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2308144189998984, "nonpadding_tokens": 13144, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23141886400003386, "nonpadding_tokens": 4765, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037528699999711534, "prefetch": {"data_wait_seconds": 0.02733409699995093, "consumed_batches": 2917, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3467, "tokens": 248612658, "loss_tokens": 50841, "loss": 2.2430878252788102, "lr": 0.00029859850191430186, "optimizer_lrs": [0.019906566794286793, 0.00029859850191430186], "gradient_norm": 0.4405406713485718, "seconds": 0.222759915999859, "nonpadding_tokens_per_second": 228232.2641925946, "ranks": [{"compute_seconds": 0.21796478399983243, "nonpadding_tokens": 3309, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.21609336699998494, "nonpadding_tokens": 10108, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.21663488300009703, "nonpadding_tokens": 12527, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.21987162499999613, "nonpadding_tokens": 4513, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.21570764499983852, "nonpadding_tokens": 9156, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.21847629000012603, "nonpadding_tokens": 4970, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2155851080001412, "nonpadding_tokens": 2951, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.22038492399997267, "nonpadding_tokens": 3307, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003934500000468688, "prefetch": {"data_wait_seconds": 0.02734271899994667, "consumed_batches": 2918, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3468, "tokens": 248656646, "loss_tokens": 43988, "loss": 2.2473790792375192, "lr": 0.0002985979634656628, "optimizer_lrs": [0.019906530897710852, 0.0002985979634656628], "gradient_norm": 0.4421098530292511, "seconds": 0.37063554500014106, "nonpadding_tokens_per_second": 118682.62662174848, "ranks": [{"compute_seconds": 0.3061752919998071, "nonpadding_tokens": 5832, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3064315849999275, "nonpadding_tokens": 3789, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3036050310001883, "nonpadding_tokens": 1386, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.30447287799984224, "nonpadding_tokens": 2859, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30692678500008697, "nonpadding_tokens": 17222, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.30412431500008097, "nonpadding_tokens": 4721, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.30271574700009296, "nonpadding_tokens": 5315, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.303399435999836, "nonpadding_tokens": 2864, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.012684331999935239, "prefetch": {"data_wait_seconds": 0.0273506049999952, "consumed_batches": 2919, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3469, "tokens": 248730389, "loss_tokens": 73743, "loss": 2.023374252471421, "lr": 0.00029859706056100467, "optimizer_lrs": [0.01990647070406698, 0.00029859706056100467], "gradient_norm": 0.48300009965896606, "seconds": 0.2632845919999909, "nonpadding_tokens_per_second": 280088.5514789356, "ranks": [{"compute_seconds": 0.25898944200002916, "nonpadding_tokens": 4283, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25987234299986994, "nonpadding_tokens": 5791, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25697536699999546, "nonpadding_tokens": 7716, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25991348599995945, "nonpadding_tokens": 9815, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2556873569999425, "nonpadding_tokens": 23419, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2583996920000118, "nonpadding_tokens": 4367, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.255674031000126, "nonpadding_tokens": 8574, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26090219799993974, "nonpadding_tokens": 9778, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00041612199993323884, "prefetch": {"data_wait_seconds": 0.02736026099996991, "consumed_batches": 2920, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3470, "tokens": 248777639, "loss_tokens": 47250, "loss": 2.223282242063492, "lr": 0.0002985964818829994, "optimizer_lrs": [0.019906432125533297, 0.0002985964818829994], "gradient_norm": 0.51117342710495, "seconds": 0.2641845580001245, "nonpadding_tokens_per_second": 178852.24010700022, "ranks": [{"compute_seconds": 0.2613667819998682, "nonpadding_tokens": 11568, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26123063899990484, "nonpadding_tokens": 3700, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2606695800000125, "nonpadding_tokens": 10730, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25924149299999044, "nonpadding_tokens": 5949, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2618354180001461, "nonpadding_tokens": 3746, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2604648899998665, "nonpadding_tokens": 3106, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2614581969999108, "nonpadding_tokens": 5967, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2581942120000349, "nonpadding_tokens": 2484, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037994499984961294, "prefetch": {"data_wait_seconds": 0.027368723000066097, "consumed_batches": 2921, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3471, "tokens": 248821735, "loss_tokens": 44096, "loss": 2.123117211595156, "lr": 0.0002985959417254398, "optimizer_lrs": [0.01990639611502932, 0.0002985959417254398], "gradient_norm": 0.4138273596763611, "seconds": 0.28049372600003153, "nonpadding_tokens_per_second": 157208.50740167729, "ranks": [{"compute_seconds": 0.27823720100013816, "nonpadding_tokens": 3440, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27741271499985487, "nonpadding_tokens": 5405, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27458481999997275, "nonpadding_tokens": 9874, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2774784609998733, "nonpadding_tokens": 5325, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.27329652999992504, "nonpadding_tokens": 3536, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.27585582599999725, "nonpadding_tokens": 2921, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2771739110000908, "nonpadding_tokens": 6763, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.27803062399993905, "nonpadding_tokens": 6832, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00036455600002227584, "prefetch": {"data_wait_seconds": 0.027376850000067066, "consumed_batches": 2922, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3472, "tokens": 248924012, "loss_tokens": 102277, "loss": 2.199270577695865, "lr": 0.00029859468847699786, "optimizer_lrs": [0.019906312565133193, 0.00029859468847699786], "gradient_norm": 0.5118075609207153, "seconds": 0.23852354899986494, "nonpadding_tokens_per_second": 428792.0435061861, "ranks": [{"compute_seconds": 0.236008972000036, "nonpadding_tokens": 4097, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23164558999997098, "nonpadding_tokens": 11402, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.23242331899996316, "nonpadding_tokens": 18901, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.23307132400009323, "nonpadding_tokens": 9987, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.23117485100010526, "nonpadding_tokens": 24983, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.234624507000035, "nonpadding_tokens": 6184, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2317690049999328, "nonpadding_tokens": 12996, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23242446200015365, "nonpadding_tokens": 13727, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004993029999695864, "prefetch": {"data_wait_seconds": 0.02738425299999392, "consumed_batches": 2923, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3473, "tokens": 248999086, "loss_tokens": 75074, "loss": 2.478295165436769, "lr": 0.0002985937682058424, "optimizer_lrs": [0.019906251213722827, 0.0002985937682058424], "gradient_norm": 0.4906308650970459, "seconds": 0.30119666299992787, "nonpadding_tokens_per_second": 249252.42946671683, "ranks": [{"compute_seconds": 0.2964111410001351, "nonpadding_tokens": 4558, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29512990800003536, "nonpadding_tokens": 5888, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.29551516999981686, "nonpadding_tokens": 4959, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29745015699995747, "nonpadding_tokens": 1595, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29472926499988716, "nonpadding_tokens": 3051, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.29722087999994073, "nonpadding_tokens": 18061, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29434006599990425, "nonpadding_tokens": 32768, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2988768070001697, "nonpadding_tokens": 4194, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00035117999982503534, "prefetch": {"data_wait_seconds": 0.027393679999889287, "consumed_batches": 2924, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3474, "tokens": 249035938, "loss_tokens": 36852, "loss": 2.2581832821149463, "lr": 0.00029859331635747775, "optimizer_lrs": [0.019906221090498517, 0.00029859331635747775], "gradient_norm": 0.5159816145896912, "seconds": 0.2611753430001045, "nonpadding_tokens_per_second": 141100.6091795781, "ranks": [{"compute_seconds": 0.25812166700006856, "nonpadding_tokens": 2133, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2551240450000023, "nonpadding_tokens": 1911, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25642137499994533, "nonpadding_tokens": 3143, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25888660999999047, "nonpadding_tokens": 1969, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2550849399999606, "nonpadding_tokens": 11605, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2579807959998561, "nonpadding_tokens": 2555, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2552729999999883, "nonpadding_tokens": 3916, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2559925650000423, "nonpadding_tokens": 9620, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037451299999702314, "prefetch": {"data_wait_seconds": 0.02740234499988503, "consumed_batches": 2925, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3475, "tokens": 249120553, "loss_tokens": 84615, "loss": 2.4323777181941737, "lr": 0.00029859227860599047, "optimizer_lrs": [0.019906151907066034, 0.00029859227860599047], "gradient_norm": 0.4610670804977417, "seconds": 0.29102522999983194, "nonpadding_tokens_per_second": 290747.9877261805, "ranks": [{"compute_seconds": 0.28864610400000856, "nonpadding_tokens": 12157, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.28878420000000915, "nonpadding_tokens": 16031, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.28624390800018773, "nonpadding_tokens": 10269, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2868352729999515, "nonpadding_tokens": 12033, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2865595129999292, "nonpadding_tokens": 16587, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2843808339998759, "nonpadding_tokens": 4800, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.28623658900005466, "nonpadding_tokens": 9517, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2858977309999773, "nonpadding_tokens": 3221, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003517069999361411, "prefetch": {"data_wait_seconds": 0.027411913999870308, "consumed_batches": 2926, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3476, "tokens": 249161237, "loss_tokens": 40684, "loss": 1.9230125786549994, "lr": 0.00029859177950597866, "optimizer_lrs": [0.01990611863373191, 0.00029859177950597866], "gradient_norm": 0.5971240401268005, "seconds": 0.22786396700007572, "nonpadding_tokens_per_second": 178545.1229328702, "ranks": [{"compute_seconds": 0.22487955900010093, "nonpadding_tokens": 2000, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.22492155899999489, "nonpadding_tokens": 5486, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2219639489999281, "nonpadding_tokens": 3453, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22487730299985742, "nonpadding_tokens": 7822, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2207661590000498, "nonpadding_tokens": 5327, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.22379663600008826, "nonpadding_tokens": 10306, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22126904900005684, "nonpadding_tokens": 2468, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2255716240001675, "nonpadding_tokens": 3822, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037712199991801754, "prefetch": {"data_wait_seconds": 0.027420583999813175, "consumed_batches": 2927, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3477, "tokens": 249240586, "loss_tokens": 79349, "loss": 1.5592793584355191, "lr": 0.00029859080582130454, "optimizer_lrs": [0.019906053721420305, 0.00029859080582130454], "gradient_norm": 0.9820165038108826, "seconds": 0.27601228799994715, "nonpadding_tokens_per_second": 287483.57754280564, "ranks": [{"compute_seconds": 0.27255090299991025, "nonpadding_tokens": 3770, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2731798559998424, "nonpadding_tokens": 12787, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.27307800199992016, "nonpadding_tokens": 2616, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.27366182199989453, "nonpadding_tokens": 3870, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2737402039999779, "nonpadding_tokens": 11770, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2714361360001476, "nonpadding_tokens": 8538, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.26886759700005314, "nonpadding_tokens": 17610, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2695299539998359, "nonpadding_tokens": 18388, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003374010000243288, "prefetch": {"data_wait_seconds": 0.027428763999708394, "consumed_batches": 2928, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3478, "tokens": 249295408, "loss_tokens": 54822, "loss": 2.291113313313998, "lr": 0.0002985901329097884, "optimizer_lrs": [0.019906008860652562, 0.0002985901329097884], "gradient_norm": 0.5087429881095886, "seconds": 0.27074061999996957, "nonpadding_tokens_per_second": 202489.00959156465, "ranks": [{"compute_seconds": 0.2667055580000124, "nonpadding_tokens": 14336, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.26434393500016995, "nonpadding_tokens": 14685, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2653246370000488, "nonpadding_tokens": 8692, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2659531679998963, "nonpadding_tokens": 3685, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.264059178000025, "nonpadding_tokens": 2225, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2671076159999757, "nonpadding_tokens": 6082, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2639539169999807, "nonpadding_tokens": 4026, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2684317439998267, "nonpadding_tokens": 1091, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003297739999652549, "prefetch": {"data_wait_seconds": 0.02743657099991026, "consumed_batches": 2929, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3479, "tokens": 249345782, "loss_tokens": 50374, "loss": 2.312633222247588, "lr": 0.00029858951445429604, "optimizer_lrs": [0.019905967630286403, 0.00029858951445429604], "gradient_norm": 0.5333157777786255, "seconds": 0.27650151300008474, "nonpadding_tokens_per_second": 182183.45156029784, "ranks": [{"compute_seconds": 0.256137826999975, "nonpadding_tokens": 11722, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.27084493600000314, "nonpadding_tokens": 3577, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26766620500006866, "nonpadding_tokens": 3827, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2679994869999973, "nonpadding_tokens": 7480, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2713265509999019, "nonpadding_tokens": 4949, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.26957177100007357, "nonpadding_tokens": 3153, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.27083509599992794, "nonpadding_tokens": 5450, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2672850550000021, "nonpadding_tokens": 10216, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003630670000802638, "prefetch": {"data_wait_seconds": 0.02744496400005403, "consumed_batches": 2930, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3480, "tokens": 249415461, "loss_tokens": 69679, "loss": 2.4584292344178302, "lr": 0.00029858865876363324, "optimizer_lrs": [0.019905910584242216, 0.00029858865876363324], "gradient_norm": 0.4385896325111389, "seconds": 0.2700228749999951, "nonpadding_tokens_per_second": 258048.50792734232, "ranks": [{"compute_seconds": 0.2659519189999173, "nonpadding_tokens": 11726, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.264207001999921, "nonpadding_tokens": 7887, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.26469327399991016, "nonpadding_tokens": 17191, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2647927410000648, "nonpadding_tokens": 4487, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2637680420000379, "nonpadding_tokens": 8419, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2665179210000588, "nonpadding_tokens": 11076, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2676527060000353, "nonpadding_tokens": 2683, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.26440018700009205, "nonpadding_tokens": 6210, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003721059999861609, "prefetch": {"data_wait_seconds": 0.02745448100013448, "consumed_batches": 2931, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3481, "tokens": 249477653, "loss_tokens": 62192, "loss": 2.191474586763571, "lr": 0.00029858789479882304, "optimizer_lrs": [0.019905859653254872, 0.00029858789479882304], "gradient_norm": 0.42420729994773865, "seconds": 0.23106167800006006, "nonpadding_tokens_per_second": 269157.5709927279, "ranks": [{"compute_seconds": 0.2286120450000908, "nonpadding_tokens": 8873, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.2250808210001196, "nonpadding_tokens": 4383, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22764124000013908, "nonpadding_tokens": 14467, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22599088700007997, "nonpadding_tokens": 11637, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22411050899995644, "nonpadding_tokens": 6539, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2271617270000661, "nonpadding_tokens": 5011, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.22872782800004643, "nonpadding_tokens": 8261, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2250666029999593, "nonpadding_tokens": 3021, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003478119999726914, "prefetch": {"data_wait_seconds": 0.027462378000294052, "consumed_batches": 2932, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3482, "tokens": 249519158, "loss_tokens": 41505, "loss": 2.291443990181906, "lr": 0.0002985873848381445, "optimizer_lrs": [0.0199058256558763, 0.0002985873848381445], "gradient_norm": 0.48054295778274536, "seconds": 0.24542549999978291, "nonpadding_tokens_per_second": 169114.45632192545, "ranks": [{"compute_seconds": 0.24235857000007854, "nonpadding_tokens": 7926, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.24264269600007538, "nonpadding_tokens": 2862, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.24160618599989903, "nonpadding_tokens": 6248, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.2424712649999492, "nonpadding_tokens": 4731, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2383751630000006, "nonpadding_tokens": 9085, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.24072794600010639, "nonpadding_tokens": 6667, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24306516100000408, "nonpadding_tokens": 1922, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2392250640000384, "nonpadding_tokens": 2064, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003868750000037835, "prefetch": {"data_wait_seconds": 0.027470768000242174, "consumed_batches": 2933, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3483, "tokens": 249578751, "loss_tokens": 59593, "loss": 1.9595967647206887, "lr": 0.0002985866524749702, "optimizer_lrs": [0.01990577683166468, 0.0002985866524749702], "gradient_norm": 0.4678856432437897, "seconds": 0.3132807079998656, "nonpadding_tokens_per_second": 190222.37398680023, "ranks": [{"compute_seconds": 0.3097849889998088, "nonpadding_tokens": 17183, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.31088057600004504, "nonpadding_tokens": 3194, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30813310600001387, "nonpadding_tokens": 3660, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3082173960001455, "nonpadding_tokens": 8539, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.30700397599980533, "nonpadding_tokens": 11026, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.3098291960000097, "nonpadding_tokens": 9756, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.3109218519998649, "nonpadding_tokens": 3142, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.30754630899991753, "nonpadding_tokens": 3093, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003508890001739928, "prefetch": {"data_wait_seconds": 0.027479074000169135, "consumed_batches": 2934, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3484, "tokens": 249620249, "loss_tokens": 41498, "loss": 2.4501962067449035, "lr": 0.0002985861423772704, "optimizer_lrs": [0.01990574282515136, 0.0002985861423772704], "gradient_norm": 0.46275943517684937, "seconds": 0.18254804000002878, "nonpadding_tokens_per_second": 227326.4615713949, "ranks": [{"compute_seconds": 0.1782928489999449, "nonpadding_tokens": 2847, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.17583267500003785, "nonpadding_tokens": 4024, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.17521104399997967, "nonpadding_tokens": 7284, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.17555751800000508, "nonpadding_tokens": 4691, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.17385081899988108, "nonpadding_tokens": 8957, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.17691259699995499, "nonpadding_tokens": 2714, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.1745595739998862, "nonpadding_tokens": 6400, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.18016598899998826, "nonpadding_tokens": 4581, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004096130001016718, "prefetch": {"data_wait_seconds": 0.02748616900016998, "consumed_batches": 2935, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3485, "tokens": 249677336, "loss_tokens": 57087, "loss": 2.4130803094399775, "lr": 0.0002985854405083772, "optimizer_lrs": [0.019905696033891813, 0.0002985854405083772], "gradient_norm": 0.49242645502090454, "seconds": 0.25861357300004784, "nonpadding_tokens_per_second": 220742.4743325032, "ranks": [{"compute_seconds": 0.2537978679999924, "nonpadding_tokens": 9498, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25605392299985397, "nonpadding_tokens": 5819, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.25508503200012456, "nonpadding_tokens": 1431, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25348064599984355, "nonpadding_tokens": 9352, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.25626984700011235, "nonpadding_tokens": 7515, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.25306189599996287, "nonpadding_tokens": 11897, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2516057559998899, "nonpadding_tokens": 5414, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2525511750000078, "nonpadding_tokens": 6161, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003737719998753164, "prefetch": {"data_wait_seconds": 0.02750948000011988, "consumed_batches": 2936, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3486, "tokens": 249721789, "loss_tokens": 44453, "loss": 2.2001395785998694, "lr": 0.0002985848938510436, "optimizer_lrs": [0.019905659590069576, 0.0002985848938510436], "gradient_norm": 0.526037871837616, "seconds": 0.25631827799998064, "nonpadding_tokens_per_second": 173428.9116908134, "ranks": [{"compute_seconds": 0.2539438579999569, "nonpadding_tokens": 5634, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.25014714700000695, "nonpadding_tokens": 1998, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2502695510002013, "nonpadding_tokens": 7471, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.25293016000000534, "nonpadding_tokens": 7322, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.2534306939999169, "nonpadding_tokens": 7164, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2518971680001414, "nonpadding_tokens": 6713, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.24937415199997304, "nonpadding_tokens": 5133, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.2499573639997834, "nonpadding_tokens": 3018, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00040395199994236464, "prefetch": {"data_wait_seconds": 0.027528950000217378, "consumed_batches": 2937, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3487, "tokens": 249765258, "loss_tokens": 43469, "loss": 2.1501434571188662, "lr": 0.0002985843591927853, "optimizer_lrs": [0.019905623946185686, 0.0002985843591927853], "gradient_norm": 1.4521405696868896, "seconds": 0.23487604299998566, "nonpadding_tokens_per_second": 185072.08928073884, "ranks": [{"compute_seconds": 0.23248904999991282, "nonpadding_tokens": 2893, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.23209248399984972, "nonpadding_tokens": 11374, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.22873475999995208, "nonpadding_tokens": 3082, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.22904008999989856, "nonpadding_tokens": 9180, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.22821240400003262, "nonpadding_tokens": 2252, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.230384051999863, "nonpadding_tokens": 7659, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.2281311909998749, "nonpadding_tokens": 3412, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.23228789600011623, "nonpadding_tokens": 3617, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0004076180000538443, "prefetch": {"data_wait_seconds": 0.027539270000218607, "consumed_batches": 2938, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3488, "tokens": 249806943, "loss_tokens": 41685, "loss": 2.521506162288593, "lr": 0.0002985838463829463, "optimizer_lrs": [0.019905589758863088, 0.0002985838463829463], "gradient_norm": 0.43832165002822876, "seconds": 0.2968192870000621, "nonpadding_tokens_per_second": 140438.98703924613, "ranks": [{"compute_seconds": 0.2942791440000292, "nonpadding_tokens": 13970, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.29456908399993154, "nonpadding_tokens": 3586, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.2937682300000688, "nonpadding_tokens": 3744, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.29430543600005876, "nonpadding_tokens": 2645, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29071397899997464, "nonpadding_tokens": 1455, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2926900170000408, "nonpadding_tokens": 4816, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29010388999995484, "nonpadding_tokens": 8374, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.29114950100006354, "nonpadding_tokens": 3095, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003911290000360168, "prefetch": {"data_wait_seconds": 0.027549277000119332, "consumed_batches": 2939, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3489, "tokens": 249872623, "loss_tokens": 65680, "loss": 2.1588499638398293, "lr": 0.00029858303819862026, "optimizer_lrs": [0.01990553587990802, 0.00029858303819862026], "gradient_norm": 0.7392056584358215, "seconds": 0.2035702579998997, "nonpadding_tokens_per_second": 322640.4517305881, "ranks": [{"compute_seconds": 0.20072358900006293, "nonpadding_tokens": 2531, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.20132768000007673, "nonpadding_tokens": 5900, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.19839341500005503, "nonpadding_tokens": 10152, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.19883910000021388, "nonpadding_tokens": 13379, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.19702150800003437, "nonpadding_tokens": 12223, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.2004841950001719, "nonpadding_tokens": 2808, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.19734957500008932, "nonpadding_tokens": 10098, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.19778356500000882, "nonpadding_tokens": 8589, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000348022000025594, "prefetch": {"data_wait_seconds": 0.027557282000046257, "consumed_batches": 2940, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3490, "tokens": 249930141, "loss_tokens": 57518, "loss": 2.1802562078392853, "lr": 0.0002985823302583491, "optimizer_lrs": [0.01990548868388994, 0.0002985823302583491], "gradient_norm": 0.5341259241104126, "seconds": 0.3241894150000917, "nonpadding_tokens_per_second": 177420.96854082585, "ranks": [{"compute_seconds": 0.31852195800001937, "nonpadding_tokens": 8103, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.3211969550000049, "nonpadding_tokens": 1938, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.3175980169999093, "nonpadding_tokens": 4374, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3205711690000044, "nonpadding_tokens": 7457, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.3173720409999987, "nonpadding_tokens": 11949, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.32021992399995725, "nonpadding_tokens": 3539, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.32100069999978587, "nonpadding_tokens": 15943, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.32188307699993857, "nonpadding_tokens": 4215, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.0003384159999768599, "prefetch": {"data_wait_seconds": 0.027565035000137073, "consumed_batches": 2941, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3491, "tokens": 249994559, "loss_tokens": 64418, "loss": 1.8833555838430251, "lr": 0.000298581537183046, "optimizer_lrs": [0.019905435812203068, 0.000298581537183046], "gradient_norm": 0.49256303906440735, "seconds": 0.30561766100004206, "nonpadding_tokens_per_second": 210779.7035983177, "ranks": [{"compute_seconds": 0.3019664199998715, "nonpadding_tokens": 11498, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30308651600012126, "nonpadding_tokens": 7048, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.30026326999995945, "nonpadding_tokens": 10588, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.3003178679998655, "nonpadding_tokens": 10692, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.29926354600002014, "nonpadding_tokens": 4941, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.300668991999828, "nonpadding_tokens": 10822, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.29897638400007054, "nonpadding_tokens": 2834, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3033696610000334, "nonpadding_tokens": 5995, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.00037844799999220413, "prefetch": {"data_wait_seconds": 0.027572877000196172, "consumed_batches": 2942, "ready_batches": 4, "buffer_capacity": 4}}
-{"step": 3492, "tokens": 250051061, "loss_tokens": 56502, "loss": 2.235658140419808, "lr": 0.0002985808413831269, "optimizer_lrs": [0.019905389425541795, 0.0002985808413831269], "gradient_norm": 0.411222904920578, "seconds": 0.31546060699997724, "nonpadding_tokens_per_second": 179109.52666113418, "ranks": [{"compute_seconds": 0.31192402099986793, "nonpadding_tokens": 5574, "peak_allocated_bytes": 15012812288, "peak_reserved_bytes": 18312331264}, {"compute_seconds": 0.30972465900003954, "nonpadding_tokens": 2421, "peak_allocated_bytes": 14991057408, "peak_reserved_bytes": 18555600896}, {"compute_seconds": 0.31221408499982317, "nonpadding_tokens": 5883, "peak_allocated_bytes": 15001686528, "peak_reserved_bytes": 18362662912}, {"compute_seconds": 0.31030743199994504, "nonpadding_tokens": 15908, "peak_allocated_bytes": 14999719424, "peak_reserved_bytes": 17871929344}, {"compute_seconds": 0.31313083399982133, "nonpadding_tokens": 2873, "peak_allocated_bytes": 14981928448, "peak_reserved_bytes": 18553503744}, {"compute_seconds": 0.31029873900001803, "nonpadding_tokens": 7684, "peak_allocated_bytes": 14995985920, "peak_reserved_bytes": 19063111680}, {"compute_seconds": 0.31312715700005356, "nonpadding_tokens": 10553, "peak_allocated_bytes": 14989914624, "peak_reserved_bytes": 17632854016}, {"compute_seconds": 0.3097385949999989, "nonpadding_tokens": 5606, "peak_allocated_bytes": 15002307072, "peak_reserved_bytes": 17874026496}], "data_fetch_seconds": 0.000376697000092463, "prefetch": {"data_wait_seconds": 0.027582439000255476, "consumed_batches": 2943, "ready_batches": 4, "buffer_capacity": 4}}
+version https://git-lfs.github.com/spec/v1
+oid sha256:321b1ea178fa5880721a9ddf774532ede32f5b90a5ea8e7e9e38d7072fa9c840
+size 12478127