logs

logs/loss.jsonl

@@ -521,3 +521,65 @@
 {"log_index": 520, "loss": 0.0191, "grad_norm": 0.20326630771160126, "learning_rate": 5.067790185483522e-05}
 {"log_index": 521, "loss": 0.021, "grad_norm": 0.15432673692703247, "learning_rate": 5.0512566665464844e-05}
 {"log_index": 522, "loss": 0.0182, "grad_norm": 0.1974451094865799, "learning_rate": 5.034722587074755e-05}
+{"log_index": 523, "loss": 0.0192, "grad_norm": 0.15378537774085999, "learning_rate": 5.018188127882375e-05}
+{"log_index": 524, "loss": 0.0217, "grad_norm": 0.11713356524705887, "learning_rate": 5.0016534697875417e-05}
+{"log_index": 525, "loss": 0.0211, "grad_norm": 0.17563804984092712, "learning_rate": 4.9851187936106294e-05}
+{"log_index": 526, "loss": 0.0199, "grad_norm": 0.12298727035522461, "learning_rate": 4.968584280172206e-05}
+{"log_index": 527, "loss": 0.0172, "grad_norm": 0.1557370126247406, "learning_rate": 4.95205011029106e-05}
+{"log_index": 528, "loss": 0.018, "grad_norm": 0.17105884850025177, "learning_rate": 4.935516464782227e-05}
+{"log_index": 529, "loss": 0.0192, "grad_norm": 0.19800813496112823, "learning_rate": 4.918983524455003e-05}
+{"log_index": 530, "loss": 0.0172, "grad_norm": 0.15134823322296143, "learning_rate": 4.9024514701109766e-05}
+{"log_index": 531, "loss": 0.0192, "grad_norm": 0.14754195511341095, "learning_rate": 4.885920482542043e-05}
+{"log_index": 532, "loss": 0.0205, "grad_norm": 0.17444182932376862, "learning_rate": 4.869390742528438e-05}
+{"log_index": 533, "loss": 0.0192, "grad_norm": 0.19070789217948914, "learning_rate": 4.852862430836744e-05}
+{"log_index": 534, "loss": 0.0186, "grad_norm": 0.22498704493045807, "learning_rate": 4.836335728217933e-05}
+{"log_index": 535, "loss": 0.0181, "grad_norm": 0.15300312638282776, "learning_rate": 4.819810815405379e-05}
+{"log_index": 536, "loss": 0.0166, "grad_norm": 0.18098650872707367, "learning_rate": 4.803287873112877e-05}
+{"log_index": 537, "loss": 0.0174, "grad_norm": 0.17039719223976135, "learning_rate": 4.786767082032681e-05}
+{"log_index": 538, "loss": 0.0172, "grad_norm": 0.14789973199367523, "learning_rate": 4.77024862283351e-05}
+{"log_index": 539, "loss": 0.0176, "grad_norm": 0.16846291720867157, "learning_rate": 4.753732676158593e-05}
+{"log_index": 540, "loss": 0.0161, "grad_norm": 0.17277640104293823, "learning_rate": 4.737219422623672e-05}
+{"log_index": 541, "loss": 0.0183, "grad_norm": 0.1361333578824997, "learning_rate": 4.720709042815044e-05}
+{"log_index": 542, "loss": 0.0167, "grad_norm": 0.12257444858551025, "learning_rate": 4.704201717287578e-05}
+{"log_index": 543, "loss": 0.0153, "grad_norm": 0.163507342338562, "learning_rate": 4.6876976265627404e-05}
+{"log_index": 544, "loss": 0.0166, "grad_norm": 0.1412845253944397, "learning_rate": 4.671196951126626e-05}
+{"log_index": 545, "loss": 0.016, "grad_norm": 0.14406627416610718, "learning_rate": 4.654699871427971e-05}
+{"log_index": 546, "loss": 0.0173, "grad_norm": 0.1475248485803604, "learning_rate": 4.6382065678762034e-05}
+{"log_index": 547, "loss": 0.0161, "grad_norm": 0.14322565495967865, "learning_rate": 4.6217172208394424e-05}
+{"log_index": 548, "loss": 0.017, "grad_norm": 0.18819501996040344, "learning_rate": 4.605232010642549e-05}
+{"log_index": 549, "loss": 0.0202, "grad_norm": 0.17882366478443146, "learning_rate": 4.588751117565142e-05}
+{"log_index": 550, "loss": 0.0152, "grad_norm": 0.13843999803066254, "learning_rate": 4.5722747218396214e-05}
+{"log_index": 551, "loss": 0.0172, "grad_norm": 0.1483648270368576, "learning_rate": 4.5558030036492194e-05}
+{"log_index": 552, "loss": 0.0184, "grad_norm": 0.14597655832767487, "learning_rate": 4.539336143125999e-05}
+{"log_index": 553, "loss": 0.0185, "grad_norm": 0.16897429525852203, "learning_rate": 4.522874320348916e-05}
+{"log_index": 554, "loss": 0.0187, "grad_norm": 0.1432882696390152, "learning_rate": 4.506417715341821e-05}
+{"log_index": 555, "loss": 0.0186, "grad_norm": 0.14911118149757385, "learning_rate": 4.489966508071511e-05}
+{"log_index": 556, "loss": 0.0203, "grad_norm": 0.164886474609375, "learning_rate": 4.4735208784457575e-05}
+{"log_index": 557, "loss": 0.0207, "grad_norm": 0.19501349329948425, "learning_rate": 4.457081006311325e-05}
+{"log_index": 558, "loss": 0.019, "grad_norm": 0.1577385514974594, "learning_rate": 4.440647071452027e-05}
+{"log_index": 559, "loss": 0.0218, "grad_norm": 0.19377802312374115, "learning_rate": 4.424219253586737e-05}
+{"log_index": 560, "loss": 0.0188, "grad_norm": 0.17058779299259186, "learning_rate": 4.407797732367443e-05}
+{"log_index": 561, "loss": 0.018, "grad_norm": 0.18023595213890076, "learning_rate": 4.391382687377268e-05}
+{"log_index": 562, "loss": 0.0177, "grad_norm": 0.15269461274147034, "learning_rate": 4.374974298128512e-05}
+{"log_index": 563, "loss": 0.0169, "grad_norm": 0.14242444932460785, "learning_rate": 4.358572744060699e-05}
+{"log_index": 564, "loss": 0.02, "grad_norm": 0.16857141256332397, "learning_rate": 4.342178204538588e-05}
+{"log_index": 565, "loss": 0.018, "grad_norm": 0.1463155448436737, "learning_rate": 4.325790858850241e-05}
+{"log_index": 566, "loss": 0.0201, "grad_norm": 0.2303977906703949, "learning_rate": 4.309410886205043e-05}
+{"log_index": 567, "loss": 0.0178, "grad_norm": 0.1428903192281723, "learning_rate": 4.293038465731752e-05}
+{"log_index": 568, "loss": 0.0181, "grad_norm": 0.12199199944734573, "learning_rate": 4.276673776476533e-05}
+{"log_index": 569, "loss": 0.0151, "grad_norm": 0.15176421403884888, "learning_rate": 4.260316997401007e-05}
+{"log_index": 570, "loss": 0.0151, "grad_norm": 0.1162976622581482, "learning_rate": 4.243968307380293e-05}
+{"log_index": 571, "loss": 0.0156, "grad_norm": 0.1573445051908493, "learning_rate": 4.22762788520104e-05}
+{"log_index": 572, "loss": 0.0202, "grad_norm": 0.13958893716335297, "learning_rate": 4.211295909559491e-05}
+{"log_index": 573, "loss": 0.0177, "grad_norm": 0.14522308111190796, "learning_rate": 4.194972559059511e-05}
+{"log_index": 574, "loss": 0.0159, "grad_norm": 0.15978920459747314, "learning_rate": 4.178658012210651e-05}
+{"log_index": 575, "loss": 0.0189, "grad_norm": 0.1638321727514267, "learning_rate": 4.162352447426177e-05}
+{"log_index": 576, "loss": 0.0179, "grad_norm": 0.17740416526794434, "learning_rate": 4.146056043021135e-05}
+{"log_index": 577, "loss": 0.0174, "grad_norm": 0.16047875583171844, "learning_rate": 4.1297689772103944e-05}
+{"log_index": 578, "loss": 0.0158, "grad_norm": 0.1782425194978714, "learning_rate": 4.113491428106694e-05}
+{"log_index": 579, "loss": 0.0171, "grad_norm": 0.17610391974449158, "learning_rate": 4.0972235737187055e-05}
+{"log_index": 580, "loss": 0.018, "grad_norm": 0.19713976979255676, "learning_rate": 4.080965591949076e-05}
+{"log_index": 581, "loss": 0.0179, "grad_norm": 0.1527530699968338, "learning_rate": 4.0647176605924924e-05}
+{"log_index": 582, "loss": 0.0173, "grad_norm": 0.13567110896110535, "learning_rate": 4.0484799573337255e-05}
+{"log_index": 583, "loss": 0.0164, "grad_norm": 0.17504696547985077, "learning_rate": 4.032252659745699e-05}
+{"log_index": 584, "loss": 0.0167, "grad_norm": 0.14120298624038696, "learning_rate": 4.016035945287539e-05}

logs/train.log

@@ -1242,4 +1242,146 @@ Rank 0, Worker 6: Caching shard...
 
 52%|█████▏    | 5207/10000 [1:24:42<1:17:18,  1.03it/s]
 52%|█████▏    | 5208/10000 [1:24:43<1:17:10,  1.03it/s]
 52%|█████▏    | 5209/10000 [1:24:44<1:17:06,  1.04it/s]
 52%|█████▏    | 5210/10000 [1:24:45<1:17:12,  1.03it/s]
                                                        
 
 52%|█████▏    | 5210/10000 [1:24:45<1:17:12,  1.03it/s]
 52%|█████▏    | 5211/10000 [1:24:46<1:17:14,  1.03it/s]
 52%|█████▏    | 5212/10000 [1:24:47<1:17:10,  1.03it/s]
 52%|█████▏    | 5213/10000 [1:24:48<1:17:14,  1.03it/s]
 52%|█████▏    | 5214/10000 [1:24:49<1:17:03,  1.04it/s]
 52%|█████▏    | 5215/10000 [1:24:50<1:17:00,  1.04it/s]
 52%|█████▏    | 5216/10000 [1:24:51<1:17:04,  1.03it/s]
 52%|█████▏    | 5217/10000 [1:24:52<1:16:57,  1.04it/s]
 52%|█████▏    | 5218/10000 [1:24:53<1:16:51,  1.04it/s]
 52%|█████▏    | 5219/10000 [1:24:54<1:16:47,  1.04it/s]
 52%|█████▏    | 5220/10000 [1:24:55<1:16:38,  1.04it/s]
                                                        
 
 52%|█████▏    | 5220/10000 [1:24:55<1:16:38,  1.04it/s]
 52%|█████▏    | 5221/10000 [1:24:56<1:16:45,  1.04it/s]
 52%|█████▏    | 5222/10000 [1:24:56<1:16:41,  1.04it/s]
 52%|█████▏    | 5223/10000 [1:24:57<1:16:47,  1.04it/s]
 52%|█████▏    | 5224/10000 [1:24:58<1:17:05,  1.03it/s]
 52%|█████▏    | 5225/10000 [1:24:59<1:16:55,  1.03it/s]
 52%|█████▏    | 5226/10000 [1:25:00<1:16:45,  1.04it/s]
 52%|█████▏    | 5227/10000 [1:25:01<1:16:41,  1.04it/s]
 52%|█████▏    | 5228/10000 [1:25:02<1:16:30,  1.04it/s]
 52%|█████▏    | 5229/10000 [1:25:03<1:16:27,  1.04it/s]
 52%|█████▏    | 5230/10000 [1:25:04<1:16:23,  1.04it/s]
                                                        
-
 52%|█████▏    | 5230/10000 [1:25:04<1:16:23,  1.04it/s]
 52%|█████▏    | 5231/10000 [1:25:05<1:16:26,  1.04it/s]
 52%|█████▏    | 5232/10000 [1:25:06<1:16:18,  1.04it/s]
 52%|█████▏    | 5233/10000 [1:25:07<1:16:22,  1.04it/s]
 52%|█████▏    | 5234/10000 [1:25:08<1:16:21,  1.04it/s]
 52%|█████▏    | 5235/10000 [1:25:09<1:16:21,  1.04it/s]
+
 52%|█████▏    | 5230/10000 [1:25:04<1:16:23,  1.04it/s]
 52%|█████▏    | 5231/10000 [1:25:05<1:16:26,  1.04it/s]
 52%|█████▏    | 5232/10000 [1:25:06<1:16:18,  1.04it/s]
 52%|█████▏    | 5233/10000 [1:25:07<1:16:22,  1.04it/s]
 52%|█████▏    | 5234/10000 [1:25:08<1:16:21,  1.04it/s]
 52%|█████▏    | 5235/10000 [1:25:09<1:16:21,  1.04it/s]
 52%|█████▏    | 5236/10000 [1:25:10<1:16:17,  1.04it/s]
 52%|█████▏    | 5237/10000 [1:25:11<1:16:24,  1.04it/s]
 52%|█████▏    | 5238/10000 [1:25:12<1:16:19,  1.04it/s]
 52%|█████▏    | 5239/10000 [1:25:13<1:16:20,  1.04it/s]
 52%|█████▏    | 5240/10000 [1:25:14<1:16:13,  1.04it/s]
                                                        
+
 52%|█████▏    | 5240/10000 [1:25:14<1:16:13,  1.04it/s]
 52%|█████▏    | 5241/10000 [1:25:15<1:16:17,  1.04it/s]
 52%|█████▏    | 5242/10000 [1:25:16<1:16:12,  1.04it/s]
 52%|█████▏    | 5243/10000 [1:25:17<1:16:20,  1.04it/s]
 52%|█████▏    | 5244/10000 [1:25:18<1:16:18,  1.04it/s]
 52%|█████▏    | 5245/10000 [1:25:19<1:16:22,  1.04it/s]
 52%|█████▏    | 5246/10000 [1:25:20<1:16:17,  1.04it/s]
 52%|█████▏    | 5247/10000 [1:25:21<1:16:10,  1.04it/s]
 52%|█████▏    | 5248/10000 [1:25:22<1:16:04,  1.04it/s]
 52%|█████▏    | 5249/10000 [1:25:22<1:16:01,  1.04it/s]
 52%|█████▎    | 5250/10000 [1:25:23<1:15:59,  1.04it/s]
                                                        
+
 52%|█████▎    | 5250/10000 [1:25:24<1:15:59,  1.04it/s]
 53%|█████▎    | 5251/10000 [1:25:24<1:16:02,  1.04it/s]
 53%|█████▎    | 5252/10000 [1:25:25<1:15:58,  1.04it/s]
 53%|█████▎    | 5253/10000 [1:25:26<1:16:03,  1.04it/s]
 53%|█████▎    | 5254/10000 [1:25:27<1:15:57,  1.04it/s]
 53%|█████▎    | 5255/10000 [1:25:28<1:15:56,  1.04it/s]
 53%|█████▎    | 5256/10000 [1:25:29<1:15:48,  1.04it/s]
 53%|█████▎    | 5257/10000 [1:25:30<1:16:06,  1.04it/s]
 53%|█████▎    | 5258/10000 [1:25:31<1:16:18,  1.04it/s]
 53%|█████▎    | 5259/10000 [1:25:32<1:16:08,  1.04it/s]
 53%|█████▎    | 5260/10000 [1:25:33<1:16:00,  1.04it/s]
                                                        
+
 53%|█████▎    | 5260/10000 [1:25:33<1:16:00,  1.04it/s]
 53%|█████▎    | 5261/10000 [1:25:34<1:16:05,  1.04it/s]
 53%|█████▎    | 5262/10000 [1:25:35<1:16:02,  1.04it/s]
 53%|█████▎    | 5263/10000 [1:25:36<1:15:55,  1.04it/s]
 53%|█████▎    | 5264/10000 [1:25:37<1:15:45,  1.04it/s]
 53%|█████▎    | 5265/10000 [1:25:38<1:15:53,  1.04it/s]
 53%|█████▎    | 5266/10000 [1:25:39<1:15:51,  1.04it/s]
 53%|█████▎    | 5267/10000 [1:25:40<1:15:51,  1.04it/s]
 53%|█████▎    | 5268/10000 [1:25:41<1:15:40,  1.04it/s]
 53%|█████▎    | 5269/10000 [1:25:42<1:15:48,  1.04it/s]
 53%|█████▎    | 5270/10000 [1:25:43<1:15:46,  1.04it/s]
                                                        
+
 53%|█████▎    | 5270/10000 [1:25:43<1:15:46,  1.04it/s]
 53%|█████▎    | 5271/10000 [1:25:44<1:15:49,  1.04it/s]
 53%|█████▎    | 5272/10000 [1:25:45<1:15:37,  1.04it/s]
 53%|█████▎    | 5273/10000 [1:25:46<1:15:41,  1.04it/s]
 53%|█████▎    | 5274/10000 [1:25:46<1:15:39,  1.04it/s]
 53%|█████▎    | 5275/10000 [1:25:47<1:15:29,  1.04it/s]
 53%|█████▎    | 5276/10000 [1:25:48<1:15:33,  1.04it/s]
 53%|█████▎    | 5277/10000 [1:25:49<1:15:39,  1.04it/s]
 53%|█████▎    | 5278/10000 [1:25:50<1:15:38,  1.04it/s]
 53%|█████▎    | 5279/10000 [1:25:51<1:15:23,  1.04it/s]
 53%|█████▎    | 5280/10000 [1:25:52<1:15:25,  1.04it/s]
                                                        
+
 53%|█████▎    | 5280/10000 [1:25:52<1:15:25,  1.04it/s]
 53%|█████▎    | 5281/10000 [1:25:53<1:15:34,  1.04it/s]
 53%|█████▎    | 5282/10000 [1:25:54<1:15:31,  1.04it/s]
 53%|█████▎    | 5283/10000 [1:25:55<1:15:32,  1.04it/s]
 53%|█████▎    | 5284/10000 [1:25:56<1:15:29,  1.04it/s]
 53%|█████▎    | 5285/10000 [1:25:57<1:15:30,  1.04it/s]
 53%|█████▎    | 5286/10000 [1:25:58<1:15:19,  1.04it/s]
 53%|█████▎    | 5287/10000 [1:25:59<1:15:18,  1.04it/s]
 53%|█████▎    | 5288/10000 [1:26:00<1:15:22,  1.04it/s]
 53%|█████▎    | 5289/10000 [1:26:01<1:15:23,  1.04it/s]
 53%|█████▎    | 5290/10000 [1:26:02<1:15:18,  1.04it/s]
                                                        
+
 53%|█████▎    | 5290/10000 [1:26:02<1:15:18,  1.04it/s]
 53%|█████▎    | 5291/10000 [1:26:03<1:15:25,  1.04it/s]
 53%|█████▎    | 5292/10000 [1:26:04<1:15:59,  1.03it/s]
 53%|█████▎    | 5293/10000 [1:26:05<1:15:51,  1.03it/s]
 53%|█████▎    | 5294/10000 [1:26:06<1:15:40,  1.04it/s]
 53%|█████▎    | 5295/10000 [1:26:07<1:15:29,  1.04it/s]
 53%|█████▎    | 5296/10000 [1:26:08<1:15:25,  1.04it/s]
 53%|█████▎    | 5297/10000 [1:26:09<1:15:28,  1.04it/s]
 53%|█████▎    | 5298/10000 [1:26:10<1:15:11,  1.04it/s]
 53%|█████▎    | 5299/10000 [1:26:11<1:15:15,  1.04it/s]
 53%|█████▎    | 5300/10000 [1:26:11<1:15:13,  1.04it/s]
                                                        
+
 53%|█████▎    | 5300/10000 [1:26:12<1:15:13,  1.04it/s]
 53%|█████▎    | 5301/10000 [1:26:12<1:15:18,  1.04it/s]
 53%|█████▎    | 5302/10000 [1:26:13<1:15:10,  1.04it/s]
 53%|█████▎    | 5303/10000 [1:26:14<1:15:13,  1.04it/s]
 53%|█████▎    | 5304/10000 [1:26:15<1:15:09,  1.04it/s]
 53%|█████▎    | 5305/10000 [1:26:16<1:15:12,  1.04it/s]
 53%|█████▎    | 5306/10000 [1:26:17<1:15:15,  1.04it/s]
 53%|█████▎    | 5307/10000 [1:26:18<1:15:08,  1.04it/s]
 53%|█████▎    | 5308/10000 [1:26:19<1:15:00,  1.04it/s]
 53%|█████▎    | 5309/10000 [1:26:20<1:15:00,  1.04it/s]
 53%|█████▎    | 5310/10000 [1:26:21<1:14:55,  1.04it/s]
                                                        
+
 53%|█████▎    | 5310/10000 [1:26:21<1:14:55,  1.04it/s]
 53%|█████▎    | 5311/10000 [1:26:22<1:14:58,  1.04it/s]
 53%|█████▎    | 5312/10000 [1:26:23<1:14:56,  1.04it/s]
 53%|██��██▎    | 5313/10000 [1:26:24<1:14:55,  1.04it/s]
 53%|█████▎    | 5314/10000 [1:26:25<1:14:59,  1.04it/s]
 53%|█████▎    | 5315/10000 [1:26:26<1:14:57,  1.04it/s]
 53%|█████▎    | 5316/10000 [1:26:27<1:14:49,  1.04it/s]
 53%|█████▎    | 5317/10000 [1:26:28<1:14:54,  1.04it/s]
 53%|█████▎    | 5318/10000 [1:26:29<1:14:53,  1.04it/s]
 53%|█████▎    | 5319/10000 [1:26:30<1:14:56,  1.04it/s]
 53%|█████▎    | 5320/10000 [1:26:31<1:14:55,  1.04it/s]
                                                        
+
 53%|█████▎    | 5320/10000 [1:26:31<1:14:55,  1.04it/s]Rank 0, Worker 0: Wait for shard 120 in dataset 0 in 0.00 seconds
+Rank 0, Worker 0: Caching shard...
+
 53%|█████▎    | 5321/10000 [1:26:32<1:15:05,  1.04it/s]Rank 0, Worker 1: Wait for shard 66 in dataset 0 in 0.00 seconds
+Rank 0, Worker 1: Caching shard...
+
 53%|█████▎    | 5322/10000 [1:26:33<1:15:07,  1.04it/s]Rank 0, Worker 2: Wait for shard 21 in dataset 0 in 0.00 seconds
+Rank 0, Worker 2: Caching shard...
+
 53%|█████▎    | 5323/10000 [1:26:34<1:15:03,  1.04it/s]Rank 0, Worker 3: Wait for shard 41 in dataset 0 in 0.00 seconds
+Rank 0, Worker 3: Caching shard...
+
 53%|█████▎    | 5324/10000 [1:26:35<1:15:06,  1.04it/s]
 53%|█████▎    | 5325/10000 [1:26:36<1:15:03,  1.04it/s]
 53%|█████▎    | 5326/10000 [1:26:37<1:15:53,  1.03it/s]
 53%|█████▎    | 5327/10000 [1:26:37<1:15:46,  1.03it/s]Rank 0, Worker 7: Wait for shard 38 in dataset 0 in 0.00 seconds
+Rank 0, Worker 7: Caching shard...
+
 53%|█████▎    | 5328/10000 [1:26:38<1:15:34,  1.03it/s]
 53%|█████▎    | 5329/10000 [1:26:39<1:15:28,  1.03it/s]
 53%|█████▎    | 5330/10000 [1:26:40<1:15:27,  1.03it/s]
                                                        
+
 53%|█████▎    | 5330/10000 [1:26:41<1:15:27,  1.03it/s]
 53%|█████▎    | 5331/10000 [1:26:41<1:15:24,  1.03it/s]
 53%|█████▎    | 5332/10000 [1:26:42<1:15:26,  1.03it/s]
 53%|█████▎    | 5333/10000 [1:26:43<1:15:15,  1.03it/s]Rank 0, Worker 5: Wait for shard 74 in dataset 0 in 0.00 seconds
+Rank 0, Worker 5: Caching shard...
+
 53%|█████▎    | 5334/10000 [1:26:44<1:15:19,  1.03it/s]Rank 0, Worker 6: Wait for shard 51 in dataset 0 in 0.00 seconds
+Rank 0, Worker 6: Caching shard...
+
 53%|█████▎    | 5335/10000 [1:26:45<1:15:12,  1.03it/s]
 53%|█████▎    | 5336/10000 [1:26:46<1:15:02,  1.04it/s]
 53%|█████▎    | 5337/10000 [1:26:47<1:14:56,  1.04it/s]
 53%|█████▎    | 5338/10000 [1:26:48<1:14:56,  1.04it/s]
 53%|█████▎    | 5339/10000 [1:26:49<1:14:48,  1.04it/s]
 53%|█████▎    | 5340/10000 [1:26:50<1:14:38,  1.04it/s]
                                                        
+
 53%|█████▎    | 5340/10000 [1:26:50<1:14:38,  1.04it/s]Rank 0, Worker 4: Wait for shard 60 in dataset 0 in 0.00 seconds
+Rank 0, Worker 4: Caching shard...
+
 53%|█████▎    | 5341/10000 [1:26:51<1:14:51,  1.04it/s]
 53%|█████▎    | 5342/10000 [1:26:52<1:14:52,  1.04it/s]
 53%|█████▎    | 5343/10000 [1:26:53<1:14:53,  1.04it/s]
 53%|█████▎    | 5344/10000 [1:26:54<1:14:49,  1.04it/s]
 53%|█████▎    | 5345/10000 [1:26:55<1:14:54,  1.04it/s]
 53%|█████▎    | 5346/10000 [1:26:56<1:14:46,  1.04it/s]
 53%|█████▎    | 5347/10000 [1:26:57<1:14:43,  1.04it/s]
 53%|█████▎    | 5348/10000 [1:26:58<1:14:43,  1.04it/s]
 53%|█████▎    | 5349/10000 [1:26:59<1:14:39,  1.04it/s]
 54%|█████▎    | 5350/10000 [1:27:00<1:14:28,  1.04it/s]
                                                        
+
 54%|█████▎    | 5350/10000 [1:27:00<1:14:28,  1.04it/s]
 54%|█████▎    | 5351/10000 [1:27:01<1:14:42,  1.04it/s]
 54%|█████▎    | 5352/10000 [1:27:02<1:14:43,  1.04it/s]
 54%|█████▎    | 5353/10000 [1:27:03<1:14:43,  1.04it/s]
 54%|█████▎    | 5354/10000 [1:27:04<1:14:33,  1.04it/s]
 54%|█████▎    | 5355/10000 [1:27:04<1:14:30,  1.04it/s]
 54%|█████▎    | 5356/10000 [1:27:05<1:14:23,  1.04it/s]
 54%|█████▎    | 5357/10000 [1:27:06<1:14:20,  1.04it/s]
 54%|█████▎    | 5358/10000 [1:27:07<1:14:18,  1.04it/s]
 54%|█████▎    | 5359/10000 [1:27:08<1:14:20,  1.04it/s]
 54%|█████▎    | 5360/10000 [1:27:09<1:15:01,  1.03it/s]
                                                        
+
 54%|█████▎    | 5360/10000 [1:27:10<1:15:01,  1.03it/s]
 54%|█████▎    | 5361/10000 [1:27:10<1:14:58,  1.03it/s]
 54%|█████▎    | 5362/10000 [1:27:11<1:14:45,  1.03it/s]
 54%|█████▎    | 5363/10000 [1:27:12<1:14:39,  1.04it/s]
 54%|█████▎    | 5364/10000 [1:27:13<1:14:36,  1.04it/s]
 54%|█████▎    | 5365/10000 [1:27:14<1:14:32,  1.04it/s]
 54%|█████▎    | 5366/10000 [1:27:15<1:14:20,  1.04it/s]
 54%|█████▎    | 5367/10000 [1:27:16<1:14:13,  1.04it/s]
 54%|█████▎    | 5368/10000 [1:27:17<1:14:10,  1.04it/s]
 54%|█████▎    | 5369/10000 [1:27:18<1:14:12,  1.04it/s]
 54%|█████▎    | 5370/10000 [1:27:19<1:14:08,  1.04it/s]
                                                        
+
 54%|█████▎    | 5370/10000 [1:27:19<1:14:08,  1.04it/s]
 54%|█████▎    | 5371/10000 [1:27:20<1:14:07,  1.04it/s]
 54%|█████▎    | 5372/10000 [1:27:21<1:14:04,  1.04it/s]
 54%|█████▎    | 5373/10000 [1:27:22<1:14:09,  1.04it/s]
 54%|█████▎    | 5374/10000 [1:27:23<1:14:07,  1.04it/s]
 54%|█████▍    | 5375/10000 [1:27:24<1:14:08,  1.04it/s]
 54%|█████▍    | 5376/10000 [1:27:25<1:14:04,  1.04it/s]
 54%|█████▍    | 5377/10000 [1:27:26<1:14:11,  1.04it/s]
 54%|█████▍    | 5378/10000 [1:27:27<1:14:07,  1.04it/s]
 54%|█████▍    | 5379/10000 [1:27:28<1:14:02,  1.04it/s]
 54%|█████▍    | 5380/10000 [1:27:29<1:13:58,  1.04it/s]
                                                        
+
 54%|█████▍    | 5380/10000 [1:27:29<1:13:58,  1.04it/s]
 54%|█████▍    | 5381/10000 [1:27:30<1:14:02,  1.04it/s]
 54%|█████▍    | 5382/10000 [1:27:30<1:14:00,  1.04it/s]
 54%|█████▍    | 5383/10000 [1:27:31<1:13:54,  1.04it/s]
 54%|█████▍    | 5384/10000 [1:27:32<1:13:52,  1.04it/s]
 54%|█████▍    | 5385/10000 [1:27:33<1:13:54,  1.04it/s]
 54%|█████▍    | 5386/10000 [1:27:34<1:13:46,  1.04it/s]
 54%|█████▍    | 5387/10000 [1:27:35<1:13:45,  1.04it/s]
 54%|█████▍    | 5388/10000 [1:27:36<1:13:48,  1.04it/s]
 54%|█████▍    | 5389/10000 [1:27:37<1:13:51,  1.04it/s]
 54%|█████▍    | 5390/10000 [1:27:38<1:13:54,  1.04it/s]
                                                        
+
 54%|█████▍    | 5390/10000 [1:27:38<1:13:54,  1.04it/s]
 54%|█████▍    | 5391/10000 [1:27:39<1:13:57,  1.04it/s]
 54%|█████▍    | 5392/10000 [1:27:40<1:14:00,  1.04it/s]
 54%|█████▍    | 5393/10000 [1:27:41<1:13:57,  1.04it/s]
 54%|█████▍    | 5394/10000 [1:27:42<1:14:41,  1.03it/s]
 54%|█████▍    | 5395/10000 [1:27:43<1:14:22,  1.03it/s]
 54%|█████▍    | 5396/10000 [1:27:44<1:14:09,  1.03it/s]
 54%|█████▍    | 5397/10000 [1:27:45<1:14:00,  1.04it/s]
 54%|█████▍    | 5398/10000 [1:27:46<1:13:52,  1.04it/s]
 54%|█████▍    | 5399/10000 [1:27:47<1:13:51,  1.04it/s]
 54%|█████▍    | 5400/10000 [1:27:48<1:13:47,  1.04it/s]
                                                        
+
 54%|█████▍    | 5400/10000 [1:27:48<1:13:47,  1.04it/s]
 54%|█████▍    | 5401/10000 [1:27:49<1:13:57,  1.04it/s]
 54%|█████▍    | 5402/10000 [1:27:50<1:13:53,  1.04it/s]
 54%|█████▍    | 5403/10000 [1:27:51<1:13:48,  1.04it/s]
 54%|█████▍    | 5404/10000 [1:27:52<1:13:42,  1.04it/s]
 54%|█████▍    | 5405/10000 [1:27:53<1:13:42,  1.04it/s]
 54%|█████▍    | 5406/10000 [1:27:54<1:13:39,  1.04it/s]
 54%|█████▍    | 5407/10000 [1:27:55<1:13:35,  1.04it/s]
 54%|█████▍    | 5408/10000 [1:27:56<1:13:32,  1.04it/s]
 54%|█████▍    | 5409/10000 [1:27:56<1:13:41,  1.04it/s]
 54%|█████▍    | 5410/10000 [1:27:57<1:13:38,  1.04it/s]
                                                        
+
 54%|█████▍    | 5410/10000 [1:27:58<1:13:38,  1.04it/s]
 54%|█████▍    | 5411/10000 [1:27:58<1:13:40,  1.04it/s]
 54%|█████▍    | 5412/10000 [1:27:59<1:13:37,  1.04it/s]
 54%|█████▍    | 5413/10000 [1:28:00<1:13:42,  1.04it/s]
 54%|█████▍    | 5414/10000 [1:28:01<1:13:39,  1.04it/s]
 54%|█████▍    | 5415/10000 [1:28:02<1:13:35,  1.04it/s]
 54%|█████▍    | 5416/10000 [1:28:03<1:13:34,  1.04it/s]
 54%|█████▍    | 5417/10000 [1:28:04<1:13:32,  1.04it/s]
 54%|█████▍    | 5418/10000 [1:28:05<1:13:23,  1.04it/s]
 54%|█████▍    | 5419/10000 [1:28:06<1:13:25,  1.04it/s]
 54%|█████▍    | 5420/10000 [1:28:07<1:13:21,  1.04it/s]
                                                        
+
 54%|█████▍    | 5420/10000 [1:28:07<1:13:21,  1.04it/s]
 54%|█████▍    | 5421/10000 [1:28:08<1:13:31,  1.04it/s]
 54%|█████▍    | 5422/10000 [1:28:09<1:13:34,  1.04it/s]
 54%|█████▍    | 5423/10000 [1:28:10<1:13:32,  1.04it/s]
 54%|█████▍    | 5424/10000 [1:28:11<1:13:27,  1.04it/s]
 54%|█████▍    | 5425/10000 [1:28:12<1:13:26,  1.04it/s]
 54%|█████▍    | 5426/10000 [1:28:13<1:13:19,  1.04it/s]
 54%|█████▍    | 5427/10000 [1:28:14<1:13:15,  1.04it/s]
 54%|█████▍    | 5428/10000 [1:28:15<1:14:05,  1.03it/s]
 54%|█████▍    | 5429/10000 [1:28:16<1:13:56,  1.03it/s]
 54%|█████▍    | 5430/10000 [1:28:17<1:13:55,  1.03it/s]
                                                        
+
 54%|█████▍    | 5430/10000 [1:28:17<1:13:55,  1.03it/s]
 54%|█████▍    | 5431/10000 [1:28:18<1:13:42,  1.03it/s]
 54%|█████▍    | 5432/10000 [1:28:19<1:13:31,  1.04it/s]
 54%|█████▍    | 5433/10000 [1:28:20<1:13:29,  1.04it/s]
 54%|█████▍    | 5434/10000 [1:28:21<1:13:30,  1.04it/s]
 54%|█████▍    | 5435/10000 [1:28:22<1:13:27,  1.04it/s]
 54%|█████▍    | 5436/10000 [1:28:23<1:13:24,  1.04it/s]
 54%|█████▍    | 5437/10000 [1:28:23<1:13:24,  1.04it/s]
 54%|█████▍    | 5438/10000 [1:28:24<1:13:21,  1.04it/s]
 54%|█████▍    | 5439/10000 [1:28:25<1:13:19,  1.04it/s]
 54%|█████▍    | 5440/10000 [1:28:26<1:13:13,  1.04it/s]
                                                        
+
 54%|█████▍    | 5440/10000 [1:28:27<1:13:13,  1.04it/s]
 54%|█████▍    | 5441/10000 [1:28:27<1:13:19,  1.04it/s]
 54%|█████▍    | 5442/10000 [1:28:28<1:13:18,  1.04it/s]
 54%|█████▍    | 5443/10000 [1:28:29<1:13:17,  1.04it/s]
 54%|█████▍    | 5444/10000 [1:28:30<1:13:07,  1.04it/s]
 54%|█████▍    | 5445/10000 [1:28:31<1:13:08,  1.04it/s]
 54%|█████▍    | 5446/10000 [1:28:32<1:13:11,  1.04it/s]
 54%|█████▍    | 5447/10000 [1:28:33<1:13:13,  1.04it/s]
 54%|█████▍    | 5448/10000 [1:28:34<1:13:07,  1.04it/s]Rank 0, Worker 0: Wait for shard 85 in dataset 0 in 0.00 seconds
+Rank 0, Worker 0: Caching shard...
+
 54%|█████▍    | 5449/10000 [1:28:35<1:13:06,  1.04it/s]Rank 0, Worker 1: Wait for shard 132 in dataset 0 in 0.00 seconds
+Rank 0, Worker 1: Caching shard...
+
 55%|█████▍    | 5450/10000 [1:28:36<1:13:06,  1.04it/s]
                                                        
+
 55%|█████▍    | 5450/10000 [1:28:36<1:13:06,  1.04it/s]Rank 0, Worker 2: Wait for shard 77 in dataset 0 in 0.00 seconds
+Rank 0, Worker 2: Caching shard...
+
 55%|█████▍    | 5451/10000 [1:28:37<1:13:18,  1.03it/s]Rank 0, Worker 3: Wait for shard 31 in dataset 0 in 0.00 seconds
+Rank 0, Worker 3: Caching shard...
+
 55%|█████▍    | 5452/10000 [1:28:38<1:13:32,  1.03it/s]
 55%|█████▍    | 5453/10000 [1:28:39<1:13:29,  1.03it/s]
 55%|█████▍    | 5454/10000 [1:28:40<1:13:33,  1.03it/s]
 55%|█████▍    | 5455/10000 [1:28:41<1:13:34,  1.03it/s]Rank 0, Worker 7: Wait for shard 49 in dataset 0 in 0.00 seconds
+Rank 0, Worker 7: Caching shard...
+
 55%|█████▍    | 5456/10000 [1:28:42<1:13:29,  1.03it/s]
 55%|█████▍    | 5457/10000 [1:28:43<1:13:26,  1.03it/s]
 55%|█████▍    | 5458/10000 [1:28:44<1:13:46,  1.03it/s]
 55%|█████▍    | 5459/10000 [1:28:45<1:13:35,  1.03it/s]
 55%|█████▍    | 5460/10000 [1:28:46<1:13:33,  1.03it/s]
                                                        
+
 55%|█████▍    | 5460/10000 [1:28:46<1:13:33,  1.03it/s]
 55%|█████▍    | 5461/10000 [1:28:47<1:13:44,  1.03it/s]Rank 0, Worker 5: Wait for shard 7 in dataset 0 in 0.00 seconds
+Rank 0, Worker 5: Caching shard...
+
 55%|█████▍    | 5462/10000 [1:28:48<1:13:53,  1.02it/s]Rank 0, Worker 6: Wait for shard 4 in dataset 0 in 0.00 seconds
+Rank 0, Worker 6: Caching shard...
+
 55%|█████▍    | 5463/10000 [1:28:49<1:13:41,  1.03it/s]
 55%|█████▍    | 5464/10000 [1:28:50<1:13:35,  1.03it/s]
 55%|█████▍    | 5465/10000 [1:28:51<1:13:27,  1.03it/s]
 55%|���████▍    | 5466/10000 [1:28:52<1:13:22,  1.03it/s]
 55%|█████▍    | 5467/10000 [1:28:53<1:13:11,  1.03it/s]
 55%|█████▍    | 5468/10000 [1:28:54<1:13:03,  1.03it/s]Rank 0, Worker 4: Wait for shard 12 in dataset 0 in 0.00 seconds
+Rank 0, Worker 4: Caching shard...
+
 55%|█████▍    | 5469/10000 [1:28:54<1:13:04,  1.03it/s]
 55%|█████▍    | 5470/10000 [1:28:55<1:13:01,  1.03it/s]
                                                        
+
 55%|█████▍    | 5470/10000 [1:28:56<1:13:01,  1.03it/s]
 55%|█████▍    | 5471/10000 [1:28:56<1:13:13,  1.03it/s]
 55%|█████▍    | 5472/10000 [1:28:57<1:13:07,  1.03it/s]
 55%|█████▍    | 5473/10000 [1:28:58<1:13:12,  1.03it/s]
 55%|█████▍    | 5474/10000 [1:28:59<1:13:03,  1.03it/s]
 55%|█████▍    | 5475/10000 [1:29:00<1:13:07,  1.03it/s]
 55%|█████▍    | 5476/10000 [1:29:01<1:13:00,  1.03it/s]
 55%|█████▍    | 5477/10000 [1:29:02<1:12:58,  1.03it/s]
 55%|█████▍    | 5478/10000 [1:29:03<1:12:58,  1.03it/s]
 55%|█████▍    | 5479/10000 [1:29:04<1:12:54,  1.03it/s]
 55%|█████▍    | 5480/10000 [1:29:05<1:12:57,  1.03it/s]
                                                        
+
 55%|█████▍    | 5480/10000 [1:29:05<1:12:57,  1.03it/s]
 55%|█████▍    | 5481/10000 [1:29:06<1:13:00,  1.03it/s]
 55%|█████▍    | 5482/10000 [1:29:07<1:12:57,  1.03it/s]
 55%|█████▍    | 5483/10000 [1:29:08<1:12:47,  1.03it/s]
 55%|█████▍    | 5484/10000 [1:29:09<1:12:38,  1.04it/s]
 55%|█████▍    | 5485/10000 [1:29:10<1:12:37,  1.04it/s]
 55%|█████▍    | 5486/10000 [1:29:11<1:12:38,  1.04it/s]
 55%|█████▍    | 5487/10000 [1:29:12<1:12:31,  1.04it/s]
 55%|█████▍    | 5488/10000 [1:29:13<1:12:29,  1.04it/s]
 55%|█████▍    | 5489/10000 [1:29:14<1:12:29,  1.04it/s]
 55%|█████▍    | 5490/10000 [1:29:15<1:12:22,  1.04it/s]
                                                        
+
 55%|█████▍    | 5490/10000 [1:29:15<1:12:22,  1.04it/s]
 55%|█████▍    | 5491/10000 [1:29:16<1:12:27,  1.04it/s]
 55%|█████▍    | 5492/10000 [1:29:17<1:12:17,  1.04it/s]
 55%|█████▍    | 5493/10000 [1:29:18<1:12:23,  1.04it/s]
 55%|█████▍    | 5494/10000 [1:29:19<1:12:22,  1.04it/s]
 55%|█████▍    | 5495/10000 [1:29:20<1:12:36,  1.03it/s]
 55%|█████▍    | 5496/10000 [1:29:21<1:12:38,  1.03it/s]
 55%|█████▍    | 5497/10000 [1:29:22<1:12:40,  1.03it/s]
 55%|█████▍    | 5498/10000 [1:29:23<1:12:31,  1.03it/s]
 55%|█████▍    | 5499/10000 [1:29:23<1:12:26,  1.04it/s]
 55%|█████▌    | 5500/10000 [1:29:24<1:12:18,  1.04it/s]
                                                        
+
 55%|█████▌    | 5500/10000 [1:29:25<1:12:18,  1.04it/s]
 55%|█████▌    | 5501/10000 [1:29:25<1:12:28,  1.03it/s]
 55%|█████▌    | 5502/10000 [1:29:26<1:12:26,  1.03it/s]
 55%|█████▌    | 5503/10000 [1:29:27<1:12:17,  1.04it/s]
 55%|█████▌    | 5504/10000 [1:29:28<1:12:11,  1.04it/s]
 55%|█████▌    | 5505/10000 [1:29:29<1:12:19,  1.04it/s]
 55%|█████▌    | 5506/10000 [1:29:30<1:12:19,  1.04it/s]
 55%|█████▌    | 5507/10000 [1:29:31<1:12:13,  1.04it/s]
 55%|█████▌    | 5508/10000 [1:29:32<1:12:05,  1.04it/s]
 55%|█████▌    | 5509/10000 [1:29:33<1:12:03,  1.04it/s]
 55%|█████▌    | 5510/10000 [1:29:34<1:12:02,  1.04it/s]
                                                        
+
 55%|█████▌    | 5510/10000 [1:29:34<1:12:02,  1.04it/s]
 55%|█████▌    | 5511/10000 [1:29:35<1:12:07,  1.04it/s]
 55%|█████▌    | 5512/10000 [1:29:36<1:12:07,  1.04it/s]
 55%|█████▌    | 5513/10000 [1:29:37<1:12:07,  1.04it/s]
 55%|█████▌    | 5514/10000 [1:29:38<1:12:04,  1.04it/s]
 55%|█████▌    | 5515/10000 [1:29:39<1:12:01,  1.04it/s]
 55%|█████▌    | 5516/10000 [1:29:40<1:12:00,  1.04it/s]
 55%|█████▌    | 5517/10000 [1:29:41<1:12:05,  1.04it/s]
 55%|█████▌    | 5518/10000 [1:29:42<1:12:02,  1.04it/s]
 55%|█████▌    | 5519/10000 [1:29:43<1:11:54,  1.04it/s]
 55%|█████▌    | 5520/10000 [1:29:44<1:11:49,  1.04it/s]
                                                        
+
 55%|█████▌    | 5520/10000 [1:29:44<1:11:49,  1.04it/s]
 55%|█████▌    | 5521/10000 [1:29:45<1:11:56,  1.04it/s]
 55%|█████▌    | 5522/10000 [1:29:46<1:11:57,  1.04it/s]
 55%|█████▌    | 5523/10000 [1:29:47<1:11:52,  1.04it/s]
 55%|█████▌    | 5524/10000 [1:29:48<1:11:47,  1.04it/s]
 55%|█████▌    | 5525/10000 [1:29:49<1:11:51,  1.04it/s]
 55%|█████▌    | 5526/10000 [1:29:49<1:11:52,  1.04it/s]
 55%|█████▌    | 5527/10000 [1:29:50<1:11:48,  1.04it/s]
 55%|█████▌    | 5528/10000 [1:29:51<1:11:48,  1.04it/s]
 55%|█████▌    | 5529/10000 [1:29:52<1:11:58,  1.04it/s]
 55%|█████▌    | 5530/10000 [1:29:53<1:11:44,  1.04it/s]
                                                        
+
 55%|█████▌    | 5530/10000 [1:29:54<1:11:44,  1.04it/s]
 55%|█████▌    | 5531/10000 [1:29:54<1:11:47,  1.04it/s]
 55%|█████▌    | 5532/10000 [1:29:55<1:11:42,  1.04it/s]
 55%|█████▌    | 5533/10000 [1:29:56<1:11:42,  1.04it/s]
 55%|█████▌    | 5534/10000 [1:29:57<1:11:35,  1.04it/s]
 55%|█████▌    | 5535/10000 [1:29:58<1:11:33,  1.04it/s]
 55%|█████▌    | 5536/10000 [1:29:59<1:11:34,  1.04it/s]
 55%|█████▌    | 5537/10000 [1:30:00<1:11:38,  1.04it/s]
 55%|█████▌    | 5538/10000 [1:30:01<1:11:33,  1.04it/s]
 55%|█████▌    | 5539/10000 [1:30:02<1:11:28,  1.04it/s]
 55%|█████▌    | 5540/10000 [1:30:03<1:11:33,  1.04it/s]
                                                        
+
 55%|█████▌    | 5540/10000 [1:30:03<1:11:33,  1.04it/s]
 55%|█████▌    | 5541/10000 [1:30:04<1:11:39,  1.04it/s]
 55%|█████▌    | 5542/10000 [1:30:05<1:11:38,  1.04it/s]
 55%|█████▌    | 5543/10000 [1:30:06<1:11:29,  1.04it/s]
 55%|█████▌    | 5544/10000 [1:30:07<1:11:27,  1.04it/s]
 55%|█████▌    | 5545/10000 [1:30:08<1:11:31,  1.04it/s]
 55%|█████▌    | 5546/10000 [1:30:09<1:11:28,  1.04it/s]
 55%|█████▌    | 5547/10000 [1:30:10<1:11:27,  1.04it/s]
 55%|█████▌    | 5548/10000 [1:30:11<1:11:21,  1.04it/s]
 55%|█████▌    | 5549/10000 [1:30:12<1:11:24,  1.04it/s]
 56%|█████▌    | 5550/10000 [1:30:13<1:11:21,  1.04it/s]
                                                        
+
 56%|█████▌    | 5550/10000 [1:30:13<1:11:21,  1.04it/s]
 56%|█████▌    | 5551/10000 [1:30:14<1:11:22,  1.04it/s]
 56%|█████▌    | 5552/10000 [1:30:15<1:11:18,  1.04it/s]
 56%|█████▌    | 5553/10000 [1:30:15<1:11:21,  1.04it/s]
 56%|█████▌    | 5554/10000 [1:30:16<1:11:20,  1.04it/s]
 56%|█████▌    | 5555/10000 [1:30:17<1:11:18,  1.04it/s]
 56%|█████▌    | 5556/10000 [1:30:18<1:11:15,  1.04it/s]
 56%|█████▌    | 5557/10000 [1:30:19<1:11:17,  1.04it/s]
 56%|█████▌    | 5558/10000 [1:30:20<1:11:16,  1.04it/s]
 56%|█████▌    | 5559/10000 [1:30:21<1:11:16,  1.04it/s]
 56%|█████▌    | 5560/10000 [1:30:22<1:11:12,  1.04it/s]
                                                        
+
 56%|█████▌    | 5560/10000 [1:30:22<1:11:12,  1.04it/s]
 56%|█████▌    | 5561/10000 [1:30:23<1:11:15,  1.04it/s]
 56%|█████▌    | 5562/10000 [1:30:24<1:11:12,  1.04it/s]
 56%|█████▌    | 5563/10000 [1:30:25<1:11:22,  1.04it/s]
 56%|█████▌    | 5564/10000 [1:30:26<1:11:27,  1.03it/s]
 56%|█████▌    | 5565/10000 [1:30:27<1:11:18,  1.04it/s]
 56%|█████▌    | 5566/10000 [1:30:28<1:11:12,  1.04it/s]
 56%|█████▌    | 5567/10000 [1:30:29<1:11:04,  1.04it/s]
 56%|█████▌    | 5568/10000 [1:30:30<1:10:56,  1.04it/s]
 56%|█████▌    | 5569/10000 [1:30:31<1:10:58,  1.04it/s]
 56%|█████▌    | 5570/10000 [1:30:32<1:10:52,  1.04it/s]
                                                        
+
 56%|█████▌    | 5570/10000 [1:30:32<1:10:52,  1.04it/s]Rank 0, Worker 2: Wait for shard 19 in dataset 0 in 0.00 seconds
+Rank 0, Worker 2: Caching shard...
+
 56%|█████▌    | 5571/10000 [1:30:33<1:10:52,  1.04it/s]
 56%|█████▌    | 5572/10000 [1:30:34<1:10:55,  1.04it/s]
 56%|█████▌    | 5573/10000 [1:30:35<1:11:00,  1.04it/s]
 56%|█████▌    | 5574/10000 [1:30:36<1:10:54,  1.04it/s]
 56%|█████▌    | 5575/10000 [1:30:37<1:10:58,  1.04it/s]
 56%|█████▌    | 5576/10000 [1:30:38<1:11:01,  1.04it/s]Rank 0, Worker 0: Wait for shard 123 in dataset 0 in 0.00 seconds
+Rank 0, Worker 0: Caching shard...
+
 56%|█████▌    | 5577/10000 [1:30:39<1:11:03,  1.04it/s]Rank 0, Worker 1: Wait for shard 116 in dataset 0 in 0.00 seconds
+Rank 0, Worker 1: Caching shard...
+
 56%|█████▌    | 5578/10000 [1:30:40<1:11:04,  1.04it/s]
 56%|█████▌    | 5579/10000 [1:30:41<1:11:10,  1.04it/s]Rank 0, Worker 3: Wait for shard 134 in dataset 0 in 0.00 seconds
+Rank 0, Worker 3: Caching shard...
+
 56%|█████▌    | 5580/10000 [1:30:41<1:11:06,  1.04it/s]
                                                        
+
 56%|█████▌    | 5580/10000 [1:30:42<1:11:06,  1.04it/s]
 56%|█████▌    | 5581/10000 [1:30:42<1:11:12,  1.03it/s]
 56%|█████▌    | 5582/10000 [1:30:43<1:11:13,  1.03it/s]
 56%|█████▌    | 5583/10000 [1:30:44<1:11:09,  1.03it/s]Rank 0, Worker 7: Wait for shard 81 in dataset 0 in 0.00 seconds
+Rank 0, Worker 7: Caching shard...
+
 56%|█████▌    | 5584/10000 [1:30:45<1:11:08,  1.03it/s]
 56%|█████▌    | 5585/10000 [1:30:46<1:11:07,  1.03it/s]
 56%|█████▌    | 5586/10000 [1:30:47<1:11:25,  1.03it/s]
 56%|█████▌    | 5587/10000 [1:30:48<1:11:05,  1.03it/s]
 56%|█████▌    | 5588/10000 [1:30:49<1:11:09,  1.03it/s]Rank 0, Worker 4: Wait for shard 73 in dataset 0 in 0.00 seconds
+Rank 0, Worker 4: Caching shard...
+
 56%|█████▌    | 5589/10000 [1:30:50<1:11:08,  1.03it/s]Rank 0, Worker 5: Wait for shard 131 in dataset 0 in 0.00 seconds
+Rank 0, Worker 5: Caching shard...
+
 56%|█████▌    | 5590/10000 [1:30:51<1:11:19,  1.03it/s]
                                                        
+
 56%|█████▌    | 5590/10000 [1:30:51<1:11:19,  1.03it/s]Rank 0, Worker 6: Wait for shard 5 in dataset 0 in 0.00 seconds
+Rank 0, Worker 6: Caching shard...
+
 56%|█████▌    | 5591/10000 [1:30:52<1:11:10,  1.03it/s]
 56%|█████▌    | 5592/10000 [1:30:53<1:11:03,  1.03it/s]
 56%|█████▌    | 5593/10000 [1:30:54<1:11:07,  1.03it/s]
 56%|█████▌    | 5594/10000 [1:30:55<1:11:03,  1.03it/s]
 56%|█████▌    | 5595/10000 [1:30:56<1:10:58,  1.03it/s]
 56%|█████▌    | 5596/10000 [1:30:57<1:10:57,  1.03it/s]
 56%|█████▌    | 5597/10000 [1:30:58<1:11:05,  1.03it/s]
 56%|█████▌    | 5598/10000 [1:30:59<1:11:05,  1.03it/s]
 56%|█████▌    | 5599/10000 [1:31:00<1:10:55,  1.03it/s]
 56%|█████▌    | 5600/10000 [1:31:01<1:10:58,  1.03it/s]
                                                        
+
 56%|█████▌    | 5600/10000 [1:31:01<1:10:58,  1.03it/s]
 56%|█████▌    | 5601/10000 [1:31:02<1:10:58,  1.03it/s]
 56%|█████▌    | 5602/10000 [1:31:03<1:10:49,  1.04it/s]
 56%|█████▌    | 5603/10000 [1:31:04<1:10:39,  1.04it/s]
 56%|█████▌    | 5604/10000 [1:31:05<1:10:31,  1.04it/s]
 56%|█████▌    | 5605/10000 [1:31:06<1:10:35,  1.04it/s]
 56%|█████▌    | 5606/10000 [1:31:07<1:10:28,  1.04it/s]
 56%|█████▌    | 5607/10000 [1:31:08<1:10:24,  1.04it/s]
 56%|█████▌    | 5608/10000 [1:31:09<1:10:19,  1.04it/s]
 56%|█████▌    | 5609/10000 [1:31:10<1:10:23,  1.04it/s]
 56%|█████▌    | 5610/10000 [1:31:10<1:10:19,  1.04it/s]
                                                        
+
 56%|█████▌    | 5610/10000 [1:31:11<1:10:19,  1.04it/s]
 56%|█████▌    | 5611/10000 [1:31:11<1:10:20,  1.04it/s]
 56%|█████▌    | 5612/10000 [1:31:12<1:10:18,  1.04it/s]
 56%|█████▌    | 5613/10000 [1:31:13<1:10:20,  1.04it/s]
 56%|█████▌    | 5614/10000 [1:31:14<1:10:14,  1.04it/s]
 56%|█████▌    | 5615/10000 [1:31:15<1:10:09,  1.04it/s]
 56%|█████▌    | 5616/10000 [1:31:16<1:10:10,  1.04it/s]
 56%|█████▌    | 5617/10000 [1:31:17<1:10:13,  1.04it/s]
 56%|█████▌    | 5618/10000 [1:31:18<1:10:04,  1.04it/s]
 56%|█████▌    | 5619/10000 [1:31:19<1:10:00,  1.04it/s]
 56%|█████▌    | 5620/10000 [1:31:20<1:10:03,  1.04it/s]
                                                        
+
 56%|█████▌    | 5620/10000 [1:31:20<1:10:03,  1.04it/s]
 56%|█████▌    | 5621/10000 [1:31:21<1:10:02,  1.04it/s]
 56%|█████▌    | 5622/10000 [1:31:22<1:10:07,  1.04it/s]
 56%|█████▌    | 5623/10000 [1:31:23<1:10:03,  1.04it/s]
 56%|█████▌    | 5624/10000 [1:31:24<1:10:03,  1.04it/s]
 56%|█████▋    | 5625/10000 [1:31:25<1:10:08,  1.04it/s]
 56%|█████▋    | 5626/10000 [1:31:26<1:10:06,  1.04it/s]
 56%|█████▋    | 5627/10000 [1:31:27<1:10:14,  1.04it/s]
 56%|█████▋    | 5628/10000 [1:31:28<1:15:48,  1.04s/it]
 56%|█████▋    | 5629/10000 [1:31:29<1:13:59,  1.02s/it]
 56%|█████▋    | 5630/10000 [1:31:30<1:12:45,  1.00it/s]
                                                        
+
 56%|█████▋    | 5630/10000 [1:31:30<1:12:45,  1.00it/s]
 56%|█████▋    | 5631/10000 [1:31:31<1:12:16,  1.01it/s]
 56%|█████▋    | 5632/10000 [1:31:32<1:11:19,  1.02it/s]
 56%|█████▋    | 5633/10000 [1:31:33<1:10:53,  1.03it/s]
 56%|█████▋    | 5634/10000 [1:31:34<1:10:29,  1.03it/s]
 56%|█████▋    | 5635/10000 [1:31:35<1:10:07,  1.04it/s]
 56%|█████▋    | 5636/10000 [1:31:36<1:10:05,  1.04it/s]
 56%|█████▋    | 5637/10000 [1:31:37<1:10:02,  1.04it/s]
 56%|█████▋    | 5638/10000 [1:31:38<1:09:59,  1.04it/s]
 56%|█████▋    | 5639/10000 [1:31:39<1:09:53,  1.04it/s]
 56%|█████▋    | 5640/10000 [1:31:40<1:09:50,  1.04it/s]
                                                        
+
 56%|█████▋    | 5640/10000 [1:31:40<1:09:50,  1.04it/s]
 56%|█████▋    | 5641/10000 [1:31:41<1:09:54,  1.04it/s]
 56%|█████▋    | 5642/10000 [1:31:41<1:09:57,  1.04it/s]
 56%|█████▋    | 5643/10000 [1:31:42<1:09:42,  1.04it/s]
 56%|█████▋    | 5644/10000 [1:31:43<1:09:43,  1.04it/s]
 56%|█████▋    | 5645/10000 [1:31:44<1:09:46,  1.04it/s]
 56%|█████▋    | 5646/10000 [1:31:45<1:09:46,  1.04it/s]
 56%|█████▋    | 5647/10000 [1:31:46<1:09:49,  1.04it/s]
 56%|█████▋    | 5648/10000 [1:31:47<1:09:47,  1.04it/s]
 56%|█████▋    | 5649/10000 [1:31:48<1:09:48,  1.04it/s]
 56%|█████▋    | 5650/10000 [1:31:49<1:09:37,  1.04it/s]
                                                        
+
 56%|█████▋    | 5650/10000 [1:31:49<1:09:37,  1.04it/s]
 57%|█████▋    | 5651/10000 [1:31:50<1:09:42,  1.04it/s]
 57%|█████▋    | 5652/10000 [1:31:51<1:09:39,  1.04it/s]
 57%|█████▋    | 5653/10000 [1:31:52<1:09:47,  1.04it/s]
 57%|█████▋    | 5654/10000 [1:31:53<1:09:43,  1.04it/s]
 57%|█████▋    | 5655/10000 [1:31:54<1:09:39,  1.04it/s]
 57%|█████▋    | 5656/10000 [1:31:55<1:09:35,  1.04it/s]
 57%|█████▋    | 5657/10000 [1:31:56<1:09:37,  1.04it/s]
 57%|█████▋    | 5658/10000 [1:31:57<1:09:26,  1.04it/s]
 57%|█████▋    | 5659/10000 [1:31:58<1:09:29,  1.04it/s]
 57%|█████▋    | 5660/10000 [1:31:59<1:09:32,  1.04it/s]
                                                        
+
 57%|█████▋    | 5660/10000 [1:31:59<1:09:32,  1.04it/s]
 57%|█████▋    | 5661/10000 [1:32:00<1:09:36,  1.04it/s]
 57%|█████▋    | 5662/10000 [1:32:01<1:09:37,  1.04it/s]
 57%|█████▋    | 5663/10000 [1:32:02<1:09:27,  1.04it/s]
 57%|█████▋    | 5664/10000 [1:32:03<1:09:30,  1.04it/s]
 57%|█████▋    | 5665/10000 [1:32:04<1:09:48,  1.03it/s]
 57%|█████▋    | 5666/10000 [1:32:05<1:09:30,  1.04it/s]
 57%|█████▋    | 5667/10000 [1:32:06<1:09:24,  1.04it/s]
 57%|█████▋    | 5668/10000 [1:32:06<1:09:20,  1.04it/s]
 57%|█████▋    | 5669/10000 [1:32:07<1:09:18,  1.04it/s]
 57%|█████▋    | 5670/10000 [1:32:08<1:09:21,  1.04it/s]
                                                        
+
 57%|█████▋    | 5670/10000 [1:32:09<1:09:21,  1.04it/s]
 57%|█████▋    | 5671/10000 [1:32:09<1:09:23,  1.04it/s]
 57%|█████▋    | 5672/10000 [1:32:10<1:09:22,  1.04it/s]
 57%|█████▋    | 5673/10000 [1:32:11<1:09:28,  1.04it/s]
 57%|█████▋    | 5674/10000 [1:32:12<1:09:21,  1.04it/s]
 57%|█████▋    | 5675/10000 [1:32:13<1:09:18,  1.04it/s]
 57%|█████▋    | 5676/10000 [1:32:14<1:09:18,  1.04it/s]
 57%|█████▋    | 5677/10000 [1:32:15<1:09:15,  1.04it/s]
 57%|█████▋    | 5678/10000 [1:32:16<1:09:14,  1.04it/s]
 57%|█████▋    | 5679/10000 [1:32:17<1:09:11,  1.04it/s]
 57%|█████▋    | 5680/10000 [1:32:18<1:09:11,  1.04it/s]
                                                        
+
 57%|█████▋    | 5680/10000 [1:32:18<1:09:11,  1.04it/s]
 57%|█████▋    | 5681/10000 [1:32:19<1:09:08,  1.04it/s]
 57%|█████▋    | 5682/10000 [1:32:20<1:09:08,  1.04it/s]
 57%|█████▋    | 5683/10000 [1:32:21<1:09:03,  1.04it/s]
 57%|█████▋    | 5684/10000 [1:32:22<1:08:55,  1.04it/s]
 57%|█████▋    | 5685/10000 [1:32:23<1:09:02,  1.04it/s]
 57%|█████▋    | 5686/10000 [1:32:24<1:09:04,  1.04it/s]
 57%|█████▋    | 5687/10000 [1:32:25<1:08:59,  1.04it/s]
 57%|█████▋    | 5688/10000 [1:32:26<1:09:01,  1.04it/s]
 57%|█████▋    | 5689/10000 [1:32:27<1:09:03,  1.04it/s]
 57%|█████▋    | 5690/10000 [1:32:28<1:09:04,  1.04it/s]
                                                        
+
 57%|█████▋    | 5690/10000 [1:32:28<1:09:04,  1.04it/s]
 57%|█████▋    | 5691/10000 [1:32:29<1:09:08,  1.04it/s]
 57%|█████▋    | 5692/10000 [1:32:30<1:09:06,  1.04it/s]
 57%|█████▋    | 5693/10000 [1:32:31<1:09:07,  1.04it/s]
 57%|█████▋    | 5694/10000 [1:32:31<1:09:04,  1.04it/s]
 57%|█████▋    | 5695/10000 [1:32:32<1:09:04,  1.04it/s]
 57%|█████▋    | 5696/10000 [1:32:33<1:09:01,  1.04it/s]
 57%|█████▋    | 5697/10000 [1:32:34<1:09:07,  1.04it/s]
 57%|█████▋    | 5698/10000 [1:32:35<1:09:06,  1.04it/s]Rank 0, Worker 2: Wait for shard 82 in dataset 0 in 0.00 seconds
+Rank 0, Worker 2: Caching shard...
+
 57%|█████▋    | 5699/10000 [1:32:36<1:09:18,  1.03it/s]
 57%|█████▋    | 5700/10000 [1:32:37<1:09:06,  1.04it/s]
                                                        
+
 57%|█████▋    | 5700/10000 [1:32:37<1:09:06,  1.04it/s]
 57%|█████▋    | 5701/10000 [1:32:38<1:09:11,  1.04it/s]
 57%|█████▋    | 5702/10000 [1:32:39<1:09:01,  1.04it/s]
 57%|█████▋    | 5703/10000 [1:32:40<1:08:54,  1.04it/s]
 57%|█████▋    | 5704/10000 [1:32:41<1:08:54,  1.04it/s]Rank 0, Worker 0: Wait for shard 127 in dataset 0 in 0.00 seconds
+Rank 0, Worker 0: Caching shard...
+
 57%|█████▋    | 5705/10000 [1:32:42<1:08:53,  1.04it/s]Rank 0, Worker 1: Wait for shard 137 in dataset 0 in 0.00 seconds
+Rank 0, Worker 1: Caching shard...
+
 57%|█████▋    | 5706/10000 [1:32:43<1:08:57,  1.04it/s]
 57%|█████▋    | 5707/10000 [1:32:44<1:09:02,  1.04it/s]Rank 0, Worker 3: Wait for shard 2 in dataset 0 in 0.00 seconds
+Rank 0, Worker 3: Caching shard...
+
 57%|█████▋    | 5708/10000 [1:32:45<1:09:02,  1.04it/s]
 57%|█████▋    | 5709/10000 [1:32:46<1:09:26,  1.03it/s]
 57%|█████▋    | 5710/10000 [1:32:47<1:08:59,  1.04it/s]
                                                        
+
 57%|█████▋    | 5710/10000 [1:32:47<1:08:59,  1.04it/s]Rank 0, Worker 6: Wait for shard 39 in dataset 0 in 0.00 seconds
+Rank 0, Worker 6: Caching shard...
+
 57%|█████▋    | 5711/10000 [1:32:48<1:09:07,  1.03it/s]Rank 0, Worker 7: Wait for shard 78 in dataset 0 in 0.00 seconds
+Rank 0, Worker 7: Caching shard...
+
 57%|█████▋    | 5712/10000 [1:32:49<1:09:05,  1.03it/s]
 57%|█████▋    | 5713/10000 [1:32:50<1:09:06,  1.03it/s]
 57%|█████▋    | 5714/10000 [1:32:51<1:09:08,  1.03it/s]
 57%|█████▋    | 5715/10000 [1:32:52<1:09:06,  1.03it/s]
 57%|█████▋    | 5716/10000 [1:32:53<1:09:00,  1.03it/s]Rank 0, Worker 4: Wait for shard 92 in dataset 0 in 0.00 seconds
+Rank 0, Worker 4: Caching shard...
+
 57%|█████▋    | 5717/10000 [1:32:54<1:09:04,  1.03it/s]Rank 0, Worker 5: Wait for shard 37 in dataset 0 in 0.00 seconds
+Rank 0, Worker 5: Caching shard...
+
 57%|█████▋    | 5718/10000 [1:32:55<1:09:02,  1.03it/s]
 57%|█████▋    | 5719/10000 [1:32:56<1:09:04,  1.03it/s]
 57%|█████▋    | 5720/10000 [1:32:57<1:09:00,  1.03it/s]
                                                        
+
 57%|█████▋    | 5720/10000 [1:32:57<1:09:00,  1.03it/s]
 57%|█████▋    | 5721/10000 [1:32:58<1:09:03,  1.03it/s]
 57%|█████▋    | 5722/10000 [1:32:59<1:09:01,  1.03it/s]
 57%|█████▋    | 5723/10000 [1:32:59<1:08:53,  1.03it/s]
 57%|█████▋    | 5724/10000 [1:33:00<1:08:56,  1.03it/s]
 57%|█████▋    | 5725/10000 [1:33:01<1:08:52,  1.03it/s]
 57%|█████▋    | 5726/10000 [1:33:02<1:08:44,  1.04it/s]
 57%|█████▋    | 5727/10000 [1:33:03<1:08:41,  1.04it/s]
 57%|█████▋    | 5728/10000 [1:33:04<1:08:45,  1.04it/s]
 57%|█████▋    | 5729/10000 [1:33:05<1:08:42,  1.04it/s]
 57%|█████▋    | 5730/10000 [1:33:06<1:08:36,  1.04it/s]
                                                        
+
 57%|█████▋    | 5730/10000 [1:33:06<1:08:36,  1.04it/s]
 57%|█████▋    | 5731/10000 [1:33:07<1:08:37,  1.04it/s]
 57%|█████▋    | 5732/10000 [1:33:08<1:08:35,  1.04it/s]
 57%|█████▋    | 5733/10000 [1:33:09<1:08:53,  1.03it/s]
 57%|█████▋    | 5734/10000 [1:33:10<1:08:38,  1.04it/s]
 57%|█████▋    | 5735/10000 [1:33:11<1:08:35,  1.04it/s]
 57%|█████▋    | 5736/10000 [1:33:12<1:08:28,  1.04it/s]
 57%|█████▋    | 5737/10000 [1:33:13<1:08:26,  1.04it/s]
 57%|█████▋    | 5738/10000 [1:33:14<1:08:22,  1.04it/s]
 57%|█████▋    | 5739/10000 [1:33:15<1:08:17,  1.04it/s]
 57%|█████▋    | 5740/10000 [1:33:16<1:08:14,  1.04it/s]
                                                        
+
 57%|█████▋    | 5740/10000 [1:33:16<1:08:14,  1.04it/s]
 57%|█████▋    | 5741/10000 [1:33:17<1:08:17,  1.04it/s]
 57%|█████▋    | 5742/10000 [1:33:18<1:08:19,  1.04it/s]
 57%|█████▋    | 5743/10000 [1:33:19<1:08:19,  1.04it/s]
 57%|█████▋    | 5744/10000 [1:33:20<1:08:19,  1.04it/s]
 57%|█████▋    | 5745/10000 [1:33:21<1:08:11,  1.04it/s]
 57%|█████▋    | 5746/10000 [1:33:22<1:08:10,  1.04it/s]
 57%|█████▋    | 5747/10000 [1:33:23<1:08:10,  1.04it/s]
 57%|█████▋    | 5748/10000 [1:33:24<1:08:03,  1.04it/s]
 57%|█████▋    | 5749/10000 [1:33:25<1:08:05,  1.04it/s]
 57%|█████▊    | 5750/10000 [1:33:25<1:08:02,  1.04it/s]
                                                        
+
 57%|█████▊    | 5750/10000 [1:33:26<1:08:02,  1.04it/s]
 58%|█████▊    | 5751/10000 [1:33:26<1:08:00,  1.04it/s]
 58%|█████▊    | 5752/10000 [1:33:27<1:07:53,  1.04it/s]
 58%|█████▊    | 5753/10000 [1:33:28<1:08:00,  1.04it/s]
 58%|█████▊    | 5754/10000 [1:33:29<1:08:00,  1.04it/s]
 58%|█████▊    | 5755/10000 [1:33:30<1:08:07,  1.04it/s]
 58%|█████▊    | 5756/10000 [1:33:31<1:08:05,  1.04it/s]
 58%|█████▊    | 5757/10000 [1:33:32<1:08:07,  1.04it/s]
 58%|█████▊    | 5758/10000 [1:33:33<1:08:01,  1.04it/s]
 58%|█████▊    | 5759/10000 [1:33:34<1:07:56,  1.04it/s]
 58%|█████▊    | 5760/10000 [1:33:35<1:07:51,  1.04it/s]
                                                        
+
 58%|█████▊    | 5760/10000 [1:33:35<1:07:51,  1.04it/s]
 58%|█████▊    | 5761/10000 [1:33:36<1:08:01,  1.04it/s]
 58%|█████▊    | 5762/10000 [1:33:37<1:07:57,  1.04it/s]
 58%|█████▊    | 5763/10000 [1:33:38<1:07:52,  1.04it/s]
 58%|█████▊    | 5764/10000 [1:33:39<1:07:56,  1.04it/s]
 58%|█████▊    | 5765/10000 [1:33:40<1:07:59,  1.04it/s]
 58%|█████▊    | 5766/10000 [1:33:41<1:08:00,  1.04it/s]
 58%|█████▊    | 5767/10000 [1:33:42<1:08:07,  1.04it/s]
 58%|█████▊    | 5768/10000 [1:33:43<1:07:56,  1.04it/s]
 58%|█████▊    | 5769/10000 [1:33:44<1:07:54,  1.04it/s]
 58%|█████▊    | 5770/10000 [1:33:45<1:07:48,  1.04it/s]
                                                        
+
 58%|█████▊    | 5770/10000 [1:33:45<1:07:48,  1.04it/s]
 58%|█████▊    | 5771/10000 [1:33:46<1:07:49,  1.04it/s]
 58%|█████▊    | 5772/10000 [1:33:47<1:07:47,  1.04it/s]
 58%|█████▊    | 5773/10000 [1:33:48<1:07:48,  1.04it/s]
 58%|█████▊    | 5774/10000 [1:33:49<1:07:45,  1.04it/s]
 58%|█████▊    | 5775/10000 [1:33:50<1:07:43,  1.04it/s]
 58%|█████▊    | 5776/10000 [1:33:51<1:07:38,  1.04it/s]
 58%|█████▊    | 5777/10000 [1:33:51<1:07:43,  1.04it/s]
 58%|█████▊    | 5778/10000 [1:33:52<1:07:42,  1.04it/s]
 58%|█████▊    | 5779/10000 [1:33:53<1:07:44,  1.04it/s]
 58%|█████▊    | 5780/10000 [1:33:54<1:07:42,  1.04it/s]
                                                        
+
 58%|█████▊    | 5780/10000 [1:33:55<1:07:42,  1.04it/s]
 58%|█████▊    | 5781/10000 [1:33:55<1:07:48,  1.04it/s]
 58%|█████▊    | 5782/10000 [1:33:56<1:07:45,  1.04it/s]
 58%|█████▊    | 5783/10000 [1:33:57<1:07:46,  1.04it/s]
 58%|█████▊    | 5784/10000 [1:33:58<1:07:43,  1.04it/s]
 58%|█████▊    | 5785/10000 [1:33:59<1:07:32,  1.04it/s]
 58%|█████▊    | 5786/10000 [1:34:00<1:07:30,  1.04it/s]
 58%|█████▊    | 5787/10000 [1:34:01<1:07:29,  1.04it/s]
 58%|█████▊    | 5788/10000 [1:34:02<1:07:22,  1.04it/s]
 58%|█████▊    | 5789/10000 [1:34:03<1:07:32,  1.04it/s]
 58%|█████▊    | 5790/10000 [1:34:04<1:07:33,  1.04it/s]
                                                        
+
 58%|█████▊    | 5790/10000 [1:34:04<1:07:33,  1.04it/s]
 58%|█████▊    | 5791/10000 [1:34:05<1:07:31,  1.04it/s]
 58%|█████▊    | 5792/10000 [1:34:06<1:07:24,  1.04it/s]
 58%|█████▊    | 5793/10000 [1:34:07<1:07:27,  1.04it/s]
 58%|█████▊    | 5794/10000 [1:34:08<1:07:21,  1.04it/s]
 58%|█████▊    | 5795/10000 [1:34:09<1:07:17,  1.04it/s]
 58%|█████▊    | 5796/10000 [1:34:10<1:07:20,  1.04it/s]
 58%|█████▊    | 5797/10000 [1:34:11<1:07:22,  1.04it/s]
 58%|█████▊    | 5798/10000 [1:34:12<1:07:22,  1.04it/s]
 58%|█████▊    | 5799/10000 [1:34:13<1:07:20,  1.04it/s]
 58%|█████▊    | 5800/10000 [1:34:14<1:07:13,  1.04it/s]
                                                        
+
 58%|█████▊    | 5800/10000 [1:34:14<1:07:13,  1.04it/s]
 58%|█████▊    | 5801/10000 [1:34:15<1:07:34,  1.04it/s]
 58%|█████▊    | 5802/10000 [1:34:16<1:07:26,  1.04it/s]
 58%|█████▊    | 5803/10000 [1:34:16<1:07:22,  1.04it/s]
 58%|█████▊    | 5804/10000 [1:34:17<1:07:14,  1.04it/s]
 58%|█████▊    | 5805/10000 [1:34:18<1:07:12,  1.04it/s]
 58%|█████▊    | 5806/10000 [1:34:19<1:07:14,  1.04it/s]
 58%|█████▊    | 5807/10000 [1:34:20<1:07:14,  1.04it/s]
 58%|█████▊    | 5808/10000 [1:34:21<1:07:09,  1.04it/s]
 58%|█████▊    | 5809/10000 [1:34:22<1:07:06,  1.04it/s]
 58%|█████▊    | 5810/10000 [1:34:23<1:07:01,  1.04it/s]
                                                        
+
 58%|█████▊    | 5810/10000 [1:34:23<1:07:01,  1.04it/s]
 58%|█████▊    | 5811/10000 [1:34:24<1:07:03,  1.04it/s]
 58%|█████▊    | 5812/10000 [1:34:25<1:06:58,  1.04it/s]
 58%|█████▊    | 5813/10000 [1:34:26<1:07:00,  1.04it/s]
 58%|█████▊    | 5814/10000 [1:34:27<1:06:58,  1.04it/s]
 58%|█████▊    | 5815/10000 [1:34:28<1:06:55,  1.04it/s]
 58%|█████▊    | 5816/10000 [1:34:29<1:06:56,  1.04it/s]
 58%|█████▊    | 5817/10000 [1:34:30<1:07:00,  1.04it/s]
 58%|█████▊    | 5818/10000 [1:34:31<1:06:53,  1.04it/s]
 58%|█████▊    | 5819/10000 [1:34:32<1:06:47,  1.04it/s]
 58%|█████▊    | 5820/10000 [1:34:33<1:06:47,  1.04it/s]
                                                        
+
 58%|█████▊    | 5820/10000 [1:34:33<1:06:47,  1.04it/s]
 58%|█████▊    | 5821/10000 [1:34:34<1:06:56,  1.04it/s]
 58%|█████▊    | 5822/10000 [1:34:35<1:06:53,  1.04it/s]
 58%|█████▊    | 5823/10000 [1:34:36<1:06:48,  1.04it/s]
 58%|█████▊    | 5824/10000 [1:34:37<1:06:47,  1.04it/s]
 58%|█████▊    | 5825/10000 [1:34:38<1:06:50,  1.04it/s]
 58%|█████▊    | 5826/10000 [1:34:39<1:06:42,  1.04it/s]Rank 0, Worker 2: Wait for shard 40 in dataset 0 in 0.00 seconds
+Rank 0, Worker 2: Caching shard...
+
 58%|█████▊    | 5827/10000 [1:34:40<1:06:45,  1.04it/s]
 58%|█████▊    | 5828/10000 [1:34:40<1:06:40,  1.04it/s]
 58%|█████▊    | 5829/10000 [1:34:41<1:06:42,  1.04it/s]
 58%|█████▊    | 5830/10000 [1:34:42<1:06:41,  1.04it/s]
                                                        
+
 58%|█████▊    | 5830/10000 [1:34:43<1:06:41,  1.04it/s]
 58%|█████▊    | 5831/10000 [1:34:43<1:06:42,  1.04it/s]
 58%|█████▊    | 5832/10000 [1:34:44<1:06:41,  1.04it/s]Rank 0, Worker 0: Wait for shard 20 in dataset 0 in 0.00 seconds
+Rank 0, Worker 0: Caching shard...
+
 58%|█████▊    | 5833/10000 [1:34:45<1:06:42,  1.04it/s]Rank 0, Worker 1: Wait for shard 23 in dataset 0 in 0.00 seconds
+Rank 0, Worker 1: Caching shard...
+
 58%|█████▊    | 5834/10000 [1:34:46<1:06:42,  1.04it/s]
 58%|█████▊    | 5835/10000 [1:34:47<1:06:58,  1.04it/s]Rank 0, Worker 3: Wait for shard 57 in dataset 0 in 0.00 seconds
+Rank 0, Worker 3: Caching shard...
+
 58%|█████▊    | 5836/10000 [1:34:48<1:07:38,  1.03it/s]
 58%|█████▊    | 5837/10000 [1:34:49<1:08:10,  1.02it/s]
 58%|█████▊    | 5838/10000 [1:34:50<1:07:48,  1.02it/s]
 58%|█████▊    | 5839/10000 [1:34:51<1:07:39,  1.03it/s]Rank 0, Worker 7: Wait for shard 136 in dataset 0 in 0.00 seconds
+Rank 0, Worker 7: Caching shard...
+
 58%|█████▊    | 5840/10000 [1:34:52<1:07:27,  1.03it/s]
                                                        
+
 58%|█████▊    | 5840/10000 [1:34:52<1:07:27,  1.03it/s]
 58%|█████▊    | 5841/10000 [1:34:53<1:07:28,  1.03it/s]
 58%|█████▊    | 5842/10000 [1:34:54<1:07:14,  1.03it/s]
 58%|█████▊    | 5843/10000 [1:34:55<1:07:09,  1.03it/s]
 58%|█████▊    | 5844/10000 [1:34:56<1:07:11,  1.03it/s]Rank 0, Worker 4: Wait for shard 59 in dataset 0 in 0.00 seconds
+Rank 0, Worker 4: Caching shard...
+
 58%|█████▊    | 5845/10000 [1:34:57<1:07:02,  1.03it/s]Rank 0, Worker 5: Wait for shard 56 in dataset 0 in 0.00 seconds
+Rank 0, Worker 5: Caching shard...
+
 58%|█████▊    | 5846/10000 [1:34:58<1:06:53,  1.03it/s]Rank 0, Worker 6: Wait for shard 89 in dataset 0 in 0.00 seconds
+Rank 0, Worker 6: Caching shard...
+
 58%|█████▊    | 5847/10000 [1:34:59<1:06:57,  1.03it/s]
 58%|█████▊    | 5848/10000 [1:35:00<1:07:15,  1.03it/s]
 58%|█████▊    | 5849/10000 [1:35:01<1:07:10,  1.03it/s]
 58%|█████▊    | 5850/10000 [1:35:02<1:06:59,  1.03it/s]
                                                        
+
 58%|█████▊    | 5850/10000 [1:35:02<1:06:59,  1.03it/s]
 59%|█████▊    | 5851/10000 [1:35:03<1:07:05,  1.03it/s]
 59%|█████▊    | 5852/10000 [1:35:04<1:07:00,  1.03it/s]
 59%|█████▊    | 5853/10000 [1:35:05<1:06:56,  1.03it/s]
 59%|█████▊    | 5854/10000 [1:35:06<1:06:47,  1.03it/s]
 59%|█████▊    | 5855/10000 [1:35:07<1:06:39,  1.04it/s]
 59%|█████▊    | 5856/10000 [1:35:08<1:06:40,  1.04it/s]
 59%|█████▊    | 5857/10000 [1:35:09<1:06:42,  1.04it/s]
 59%|█████▊    | 5858/10000 [1:35:10<1:06:35,  1.04it/s]
 59%|█████▊    | 5859/10000 [1:35:11<1:06:28,  1.04it/s]

logs/uploader.log

@@ -18869,3 +18869,4 @@ ModuleNotFoundError: No module named 'huggingface_hub'
 
  ...0003-of-00003.safetensors: 100%|██████████| 2.62GB / 2.62GB            
 
  ...0002-of-00003.safetensors: 100%|██████████| 4.97GB / 4.97GB            
 [2026-11-08T09:36:32Z] uploader: uploaded /root/work/runs/asimov_n17/checkpoint-4000 -> checkpoints/checkpoint-4000
+[2026-11-08T09:36:34Z] uploader: uploaded /root/work/logs -> logs